src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp
author stefank
Tue, 02 Jul 2019 12:10:18 +0200
changeset 55553 b9171be3cf2a
parent 55536 8313c42345d5
child 55554 46579c68f7f4
permissions -rw-r--r--
8225770: ZGC: C2: Generates on_weak instead of on_strong barriers Reviewed-by: neliasso, eosterlund
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     1
/*
53223
df6cbf676c70 8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents: 52829
diff changeset
     2
 * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     4
 *
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     5
 * This code is free software; you can redistribute it and/or modify it
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     6
 * under the terms of the GNU General Public License version 2 only, as
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     7
 * published by the Free Software Foundation.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     8
 *
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     9
 * This code is distributed in the hope that it will be useful, but WITHOUT
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    12
 * version 2 for more details (a copy is included in the LICENSE file that
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    13
 * accompanied this code).
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    14
 *
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    15
 * You should have received a copy of the GNU General Public License version
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    16
 * 2 along with this work; if not, write to the Free Software Foundation,
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    18
 *
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    20
 * or visit www.oracle.com if you need additional information or have any
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    21
 * questions.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    22
 */
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    23
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    24
#include "precompiled.hpp"
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
    25
#include "opto/castnode.hpp"
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    26
#include "opto/compile.hpp"
52568
40474b7105f4 8213615: GC/C2 abstraction for escape analysis
rkennke
parents: 52471
diff changeset
    27
#include "opto/escape.hpp"
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    28
#include "opto/graphKit.hpp"
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    29
#include "opto/loopnode.hpp"
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
    30
#include "opto/machnode.hpp"
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    31
#include "opto/macro.hpp"
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
    32
#include "opto/memnode.hpp"
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
    33
#include "opto/movenode.hpp"
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    34
#include "opto/node.hpp"
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
    35
#include "opto/phase.hpp"
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
    36
#include "opto/phaseX.hpp"
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
    37
#include "opto/rootnode.hpp"
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    38
#include "opto/type.hpp"
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
    39
#include "utilities/copy.hpp"
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
    40
#include "utilities/growableArray.hpp"
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    41
#include "utilities/macros.hpp"
53407
690aed53fef0 8217342: Build failed with excluding JFR
zgu
parents: 53226
diff changeset
    42
#include "gc/z/zBarrierSet.hpp"
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    43
#include "gc/z/c2/zBarrierSetC2.hpp"
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    44
#include "gc/z/zThreadLocalData.hpp"
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    45
#include "gc/z/zBarrierSetRuntime.hpp"
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    46
53439
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
    47
ZBarrierSetC2State::ZBarrierSetC2State(Arena* comp_arena) :
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
    48
    _load_barrier_nodes(new (comp_arena) GrowableArray<LoadBarrierNode*>(comp_arena, 8,  0, NULL)) {}
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    49
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    50
int ZBarrierSetC2State::load_barrier_count() const {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    51
  return _load_barrier_nodes->length();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    52
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    53
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    54
void ZBarrierSetC2State::add_load_barrier_node(LoadBarrierNode * n) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    55
  assert(!_load_barrier_nodes->contains(n), " duplicate entry in expand list");
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    56
  _load_barrier_nodes->append(n);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    57
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    58
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    59
void ZBarrierSetC2State::remove_load_barrier_node(LoadBarrierNode * n) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    60
  // this function may be called twice for a node so check
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    61
  // that the node is in the array before attempting to remove it
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    62
  if (_load_barrier_nodes->contains(n)) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    63
    _load_barrier_nodes->remove(n);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    64
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    65
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    66
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    67
LoadBarrierNode* ZBarrierSetC2State::load_barrier_node(int idx) const {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    68
  return _load_barrier_nodes->at(idx);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    69
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    70
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    71
void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    72
  return new(comp_arena) ZBarrierSetC2State(comp_arena);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    73
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    74
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    75
ZBarrierSetC2State* ZBarrierSetC2::state() const {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    76
  return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    77
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    78
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    79
bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const {
51485
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
    80
  // 1. This step follows potential oop projections of a load barrier before expansion
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
    81
  if (node->is_Proj()) {
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
    82
    node = node->in(0);
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
    83
  }
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
    84
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
    85
  // 2. This step checks for unexpanded load barriers
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
    86
  if (node->is_LoadBarrier()) {
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
    87
    return true;
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
    88
  }
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
    89
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
    90
  // 3. This step checks for the phi corresponding to an optimized load barrier expansion
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
    91
  if (node->is_Phi()) {
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
    92
    PhiNode* phi = node->as_Phi();
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
    93
    Node* n = phi->in(1);
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
    94
    if (n != NULL && n->is_LoadBarrierSlowReg()) {
51485
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
    95
      return true;
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
    96
    }
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
    97
  }
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
    98
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
    99
  return false;
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   100
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   101
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   102
void ZBarrierSetC2::register_potential_barrier_node(Node* node) const {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   103
  if (node->is_LoadBarrier()) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   104
    state()->add_load_barrier_node(node->as_LoadBarrier());
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   105
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   106
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   107
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   108
void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   109
  if (node->is_LoadBarrier()) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   110
    state()->remove_load_barrier_node(node->as_LoadBarrier());
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   111
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   112
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   113
52224
4f2215a00ed1 8212611: Small collection of simple changes from shenandoah
roland
parents: 52036
diff changeset
   114
void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const {
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   115
  // Remove useless LoadBarrier nodes
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   116
  ZBarrierSetC2State* s = state();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   117
  for (int i = s->load_barrier_count()-1; i >= 0; i--) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   118
    LoadBarrierNode* n = s->load_barrier_node(i);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   119
    if (!useful.member(n)) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   120
      unregister_potential_barrier_node(n);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   121
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   122
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   123
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   124
52224
4f2215a00ed1 8212611: Small collection of simple changes from shenandoah
roland
parents: 52036
diff changeset
   125
void ZBarrierSetC2::enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   126
  if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) {
52224
4f2215a00ed1 8212611: Small collection of simple changes from shenandoah
roland
parents: 52036
diff changeset
   127
    igvn->_worklist.push(node);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   128
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   129
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   130
55553
b9171be3cf2a 8225770: ZGC: C2: Generates on_weak instead of on_strong barriers
stefank
parents: 55536
diff changeset
   131
const uint NoBarrier       = 0;
b9171be3cf2a 8225770: ZGC: C2: Generates on_weak instead of on_strong barriers
stefank
parents: 55536
diff changeset
   132
const uint RequireBarrier  = 1;
b9171be3cf2a 8225770: ZGC: C2: Generates on_weak instead of on_strong barriers
stefank
parents: 55536
diff changeset
   133
const uint WeakBarrier     = 2;
b9171be3cf2a 8225770: ZGC: C2: Generates on_weak instead of on_strong barriers
stefank
parents: 55536
diff changeset
   134
const uint ExpandedBarrier = 4;
b9171be3cf2a 8225770: ZGC: C2: Generates on_weak instead of on_strong barriers
stefank
parents: 55536
diff changeset
   135
b9171be3cf2a 8225770: ZGC: C2: Generates on_weak instead of on_strong barriers
stefank
parents: 55536
diff changeset
   136
static bool load_require_barrier(LoadNode* load)      { return (load->barrier_data() & RequireBarrier)  == RequireBarrier; }
b9171be3cf2a 8225770: ZGC: C2: Generates on_weak instead of on_strong barriers
stefank
parents: 55536
diff changeset
   137
static bool load_has_weak_barrier(LoadNode* load)     { return (load->barrier_data() & WeakBarrier)     == WeakBarrier; }
b9171be3cf2a 8225770: ZGC: C2: Generates on_weak instead of on_strong barriers
stefank
parents: 55536
diff changeset
   138
static bool load_has_expanded_barrier(LoadNode* load) { return (load->barrier_data() & ExpandedBarrier) == ExpandedBarrier; }
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   139
static void load_set_expanded_barrier(LoadNode* load) { return load->set_barrier_data(ExpandedBarrier); }
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   140
55553
b9171be3cf2a 8225770: ZGC: C2: Generates on_weak instead of on_strong barriers
stefank
parents: 55536
diff changeset
   141
static void load_set_barrier(LoadNode* load, bool weak) {
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   142
  if (weak) {
55553
b9171be3cf2a 8225770: ZGC: C2: Generates on_weak instead of on_strong barriers
stefank
parents: 55536
diff changeset
   143
    load->set_barrier_data(RequireBarrier | WeakBarrier);
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   144
  } else {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   145
    load->set_barrier_data(RequireBarrier);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   146
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   147
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   148
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   149
// == LoadBarrierNode ==
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   150
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   151
LoadBarrierNode::LoadBarrierNode(Compile* C,
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   152
                                 Node* c,
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   153
                                 Node* mem,
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   154
                                 Node* val,
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   155
                                 Node* adr,
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   156
                                 bool weak) :
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   157
    MultiNode(Number_of_Inputs),
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   158
    _weak(weak) {
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   159
  init_req(Control, c);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   160
  init_req(Memory, mem);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   161
  init_req(Oop, val);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   162
  init_req(Address, adr);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   163
  init_req(Similar, C->top());
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   164
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   165
  init_class_id(Class_LoadBarrier);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   166
  BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   167
  bs->register_potential_barrier_node(this);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   168
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   169
53223
df6cbf676c70 8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents: 52829
diff changeset
   170
uint LoadBarrierNode::size_of() const {
df6cbf676c70 8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents: 52829
diff changeset
   171
  return sizeof(*this);
df6cbf676c70 8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents: 52829
diff changeset
   172
}
df6cbf676c70 8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents: 52829
diff changeset
   173
54327
a4d19817609c 8157372: C2: Node::cmp() should return bool
jcbeyler
parents: 53766
diff changeset
   174
bool LoadBarrierNode::cmp(const Node& n) const {
53223
df6cbf676c70 8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents: 52829
diff changeset
   175
  ShouldNotReachHere();
54327
a4d19817609c 8157372: C2: Node::cmp() should return bool
jcbeyler
parents: 53766
diff changeset
   176
  return false;
53223
df6cbf676c70 8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents: 52829
diff changeset
   177
}
df6cbf676c70 8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents: 52829
diff changeset
   178
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   179
const Type *LoadBarrierNode::bottom_type() const {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   180
  const Type** floadbarrier = (const Type **)(Compile::current()->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   181
  Node* in_oop = in(Oop);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   182
  floadbarrier[Control] = Type::CONTROL;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   183
  floadbarrier[Memory] = Type::MEMORY;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   184
  floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   185
  return TypeTuple::make(Number_of_Outputs, floadbarrier);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   186
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   187
53223
df6cbf676c70 8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents: 52829
diff changeset
   188
const TypePtr* LoadBarrierNode::adr_type() const {
df6cbf676c70 8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents: 52829
diff changeset
   189
  ShouldNotReachHere();
df6cbf676c70 8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents: 52829
diff changeset
   190
  return NULL;
df6cbf676c70 8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents: 52829
diff changeset
   191
}
df6cbf676c70 8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents: 52829
diff changeset
   192
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   193
const Type *LoadBarrierNode::Value(PhaseGVN *phase) const {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   194
  const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   195
  const Type* val_t = phase->type(in(Oop));
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   196
  floadbarrier[Control] = Type::CONTROL;
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   197
  floadbarrier[Memory]  = Type::MEMORY;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   198
  floadbarrier[Oop]     = val_t;
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   199
  return TypeTuple::make(Number_of_Outputs, floadbarrier);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   200
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   201
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   202
bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   203
  if (phase != NULL) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   204
    return phase->is_dominator(d, n);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   205
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   206
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   207
  for (int i = 0; i < 10 && n != NULL; i++) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   208
    n = IfNode::up_one_dom(n, linear_only);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   209
    if (n == d) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   210
      return true;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   211
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   212
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   213
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   214
  return false;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   215
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   216
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   217
LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) {
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   218
  if (is_weak()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   219
    // Weak barriers can't be eliminated
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   220
    return NULL;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   221
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   222
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   223
  Node* val = in(LoadBarrierNode::Oop);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   224
  if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   225
    LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   226
    assert(lb->in(Address) == in(Address), "");
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   227
    // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   228
    if (lb->in(Oop) == in(Oop)) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   229
      return lb;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   230
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   231
    // Follow chain of load barrier through Similar edges
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   232
    while (!lb->in(Similar)->is_top()) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   233
      lb = lb->in(Similar)->in(0)->as_LoadBarrier();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   234
      assert(lb->in(Address) == in(Address), "");
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   235
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   236
    if (lb != in(Similar)->in(0)) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   237
      return lb;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   238
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   239
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   240
  for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   241
    Node* u = val->fast_out(i);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   242
    if (u != this && u->is_LoadBarrier() && u->in(Oop) == val && u->as_LoadBarrier()->has_true_uses()) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   243
      Node* this_ctrl = in(LoadBarrierNode::Control);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   244
      Node* other_ctrl = u->in(LoadBarrierNode::Control);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   245
      if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   246
        return u->as_LoadBarrier();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   247
      }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   248
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   249
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   250
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   251
  if (can_be_eliminated()) {
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   252
    return NULL;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   253
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   254
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   255
  if (!look_for_similar) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   256
    return NULL;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   257
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   258
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   259
  Node* addr = in(LoadBarrierNode::Address);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   260
  for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   261
    Node* u = addr->fast_out(i);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   262
    if (u != this && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   263
      Node* this_ctrl = in(LoadBarrierNode::Control);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   264
      Node* other_ctrl = u->in(LoadBarrierNode::Control);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   265
      if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   266
        ResourceMark rm;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   267
        Unique_Node_List wq;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   268
        wq.push(in(LoadBarrierNode::Control));
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   269
        bool ok = true;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   270
        bool dom_found = false;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   271
        for (uint next = 0; next < wq.size(); ++next) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   272
          Node *n = wq.at(next);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   273
          if (n->is_top()) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   274
            return NULL;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   275
          }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   276
          assert(n->is_CFG(), "");
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   277
          if (n->is_SafePoint()) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   278
            ok = false;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   279
            break;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   280
          }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   281
          if (n == u) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   282
            dom_found = true;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   283
            continue;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   284
          }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   285
          if (n->is_Region()) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   286
            for (uint i = 1; i < n->req(); i++) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   287
              Node* m = n->in(i);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   288
              if (m != NULL) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   289
                wq.push(m);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   290
              }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   291
            }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   292
          } else {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   293
            Node* m = n->in(0);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   294
            if (m != NULL) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   295
              wq.push(m);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   296
            }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   297
          }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   298
        }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   299
        if (ok) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   300
          assert(dom_found, "");
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   301
          return u->as_LoadBarrier();
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   302
        }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   303
        break;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   304
      }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   305
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   306
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   307
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   308
  return NULL;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   309
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   310
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   311
void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const {
50875
2217b2fc29ea 8205993: ZGC: Fix typos and incorrect indentations
pliden
parents: 50803
diff changeset
   312
  // Change to that barrier may affect a dominated barrier so re-push those
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   313
  assert(!is_weak(), "sanity");
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   314
  Node* val = in(LoadBarrierNode::Oop);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   315
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   316
  for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   317
    Node* u = val->fast_out(i);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   318
    if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   319
      Node* this_ctrl = in(Control);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   320
      Node* other_ctrl = u->in(Control);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   321
      if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   322
        igvn->_worklist.push(u);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   323
      }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   324
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   325
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   326
    Node* addr = in(LoadBarrierNode::Address);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   327
    for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   328
      Node* u = addr->fast_out(i);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   329
      if (u != this && u->is_LoadBarrier() && u->in(Similar)->is_top()) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   330
        Node* this_ctrl = in(Control);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   331
        Node* other_ctrl = u->in(Control);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   332
        if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   333
          igvn->_worklist.push(u);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   334
        }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   335
      }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   336
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   337
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   338
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   339
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   340
Node *LoadBarrierNode::Identity(PhaseGVN *phase) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   341
  LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   342
  if (dominating_barrier != NULL) {
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   343
    assert(!is_weak(), "Weak barriers cant be eliminated");
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   344
    assert(dominating_barrier->in(Oop) == in(Oop), "");
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   345
    return dominating_barrier;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   346
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   347
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   348
  return this;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   349
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   350
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   351
Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   352
  if (remove_dead_region(phase, can_reshape)) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   353
    return this;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   354
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   355
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   356
  Node *val = in(Oop);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   357
  Node *mem = in(Memory);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   358
  Node *ctrl = in(Control);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   359
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   360
  assert(val->Opcode() != Op_LoadN, "");
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   361
  assert(val->Opcode() != Op_DecodeN, "");
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   362
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   363
  if (mem->is_MergeMem()) {
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   364
    Node *new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   365
    set_req(Memory, new_mem);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   366
    if (mem->outcnt() == 0 && can_reshape) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   367
      phase->is_IterGVN()->_worklist.push(mem);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   368
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   369
    return this;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   370
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   371
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   372
  LoadBarrierNode *dominating_barrier = NULL;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   373
  if (!is_weak()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   374
    dominating_barrier = has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress());
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   375
    if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   376
      assert(in(Address) == dominating_barrier->in(Address), "");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   377
      set_req(Similar, dominating_barrier->proj_out(Oop));
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   378
      return this;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   379
    }
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   380
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   381
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   382
  bool eliminate = can_reshape && (dominating_barrier != NULL || !has_true_uses());
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   383
  if (eliminate) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   384
    if (can_reshape) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   385
      PhaseIterGVN* igvn = phase->is_IterGVN();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   386
      Node* out_ctrl = proj_out_or_null(Control);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   387
      Node* out_res = proj_out_or_null(Oop);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   388
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   389
      if (out_ctrl != NULL) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   390
        igvn->replace_node(out_ctrl, ctrl);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   391
      }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   392
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   393
      // That transformation may cause the Similar edge on the load barrier to be invalid
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   394
      fix_similar_in_uses(igvn);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   395
      if (out_res != NULL) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   396
        if (dominating_barrier != NULL) {
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   397
          assert(!is_weak(), "Sanity");
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   398
          igvn->replace_node(out_res, dominating_barrier->proj_out(Oop));
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   399
        } else {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   400
          igvn->replace_node(out_res, val);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   401
        }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   402
      }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   403
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   404
    return new ConINode(TypeInt::ZERO);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   405
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   406
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   407
  // If the Similar edge is no longer a load barrier, clear it
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   408
  Node* similar = in(Similar);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   409
  if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   410
    set_req(Similar, phase->C->top());
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   411
    return this;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   412
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   413
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   414
  if (can_reshape && !is_weak()) {
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   415
    // If this barrier is linked through the Similar edge by a
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   416
    // dominated barrier and both barriers have the same Oop field,
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   417
    // the dominated barrier can go away, so push it for reprocessing.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   418
    // We also want to avoid a barrier to depend on another dominating
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   419
    // barrier through its Similar edge that itself depend on another
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   420
    // barrier through its Similar edge and rather have the first
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   421
    // depend on the third.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   422
    PhaseIterGVN* igvn = phase->is_IterGVN();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   423
    Node* out_res = proj_out(Oop);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   424
    for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   425
      Node* u = out_res->fast_out(i);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   426
      if (u->is_LoadBarrier() && u->in(Similar) == out_res &&
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   427
          (u->in(Oop) == val || !u->in(Similar)->is_top())) {
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   428
        assert(!u->as_LoadBarrier()->is_weak(), "Sanity");
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   429
        igvn->_worklist.push(u);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   430
      }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   431
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   432
    push_dominated_barriers(igvn);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   433
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   434
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   435
  return NULL;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   436
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   437
53223
df6cbf676c70 8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents: 52829
diff changeset
   438
uint LoadBarrierNode::match_edge(uint idx) const {
df6cbf676c70 8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents: 52829
diff changeset
   439
  ShouldNotReachHere();
df6cbf676c70 8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents: 52829
diff changeset
   440
  return 0;
df6cbf676c70 8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents: 52829
diff changeset
   441
}
df6cbf676c70 8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents: 52829
diff changeset
   442
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   443
void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   444
  Node* out_res = proj_out_or_null(Oop);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   445
  if (out_res == NULL) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   446
    return;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   447
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   448
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   449
  for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   450
    Node* u = out_res->fast_out(i);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   451
    if (u->is_LoadBarrier() && u->in(Similar) == out_res) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   452
      igvn->replace_input_of(u, Similar, igvn->C->top());
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   453
      --i;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   454
      --imax;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   455
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   456
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   457
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   458
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   459
bool LoadBarrierNode::has_true_uses() const {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   460
  Node* out_res = proj_out_or_null(Oop);
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   461
  if (out_res != NULL) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   462
    for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   463
      Node *u = out_res->fast_out(i);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   464
      if (!u->is_LoadBarrier() || u->in(Similar) != out_res) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   465
        return true;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   466
      }
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   467
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   468
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   469
  return false;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   470
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   471
52424
e3d79743f57d 8212243: More gc interface tweaks for arraycopy
roland
parents: 52224
diff changeset
   472
static bool barrier_needed(C2Access& access) {
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   473
  return ZBarrierSet::barrier_needed(access.decorators(), access.type());
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   474
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   475
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   476
Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   477
  Node* p = BarrierSetC2::load_at_resolved(access, val_type);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   478
  if (!barrier_needed(access)) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   479
    return p;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   480
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   481
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   482
  bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0;
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   483
  if (p->isa_Load()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   484
    load_set_barrier(p->as_Load(), weak);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   485
  }
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   486
  return p;
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   487
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   488
52424
e3d79743f57d 8212243: More gc interface tweaks for arraycopy
roland
parents: 52224
diff changeset
   489
Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   490
                                                    Node* new_val, const Type* val_type) const {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   491
  Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   492
  LoadStoreNode* lsn = result->as_LoadStore();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   493
  if (barrier_needed(access)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   494
    lsn->set_has_barrier();
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   495
  }
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   496
  return lsn;
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   497
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   498
52424
e3d79743f57d 8212243: More gc interface tweaks for arraycopy
roland
parents: 52224
diff changeset
   499
Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   500
                                                     Node* new_val, const Type* value_type) const {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   501
  Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   502
  LoadStoreNode* lsn = result->as_LoadStore();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   503
  if (barrier_needed(access)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   504
    lsn->set_has_barrier();
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   505
  }
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   506
  return lsn;
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   507
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   508
52424
e3d79743f57d 8212243: More gc interface tweaks for arraycopy
roland
parents: 52224
diff changeset
   509
Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const {
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   510
  Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   511
  LoadStoreNode* lsn = result->as_LoadStore();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   512
  if (barrier_needed(access)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   513
    lsn->set_has_barrier();
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   514
  }
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   515
  return lsn;
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   516
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   517
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   518
// == Macro Expansion ==
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   519
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   520
// Optimized, low spill, loadbarrier variant using stub specialized on register used
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   521
void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   522
  PhaseIterGVN &igvn = phase->igvn();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   523
  float unlikely  = PROB_UNLIKELY(0.999);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   524
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   525
  Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   526
  Node* in_mem = barrier->in(LoadBarrierNode::Memory);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   527
  Node* in_val = barrier->in(LoadBarrierNode::Oop);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   528
  Node* in_adr = barrier->in(LoadBarrierNode::Address);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   529
55536
8313c42345d5 8226287: Make process_users_of_allocation handle gc barriers
neliasso
parents: 55326
diff changeset
   530
  Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   531
  Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   532
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   533
  assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null");
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   534
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   535
  Node* jthread = igvn.transform(new ThreadLocalNode());
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   536
  Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   537
  Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr,
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   538
                                                 TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(),
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   539
                                                 MemNode::unordered));
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   540
  Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   541
  Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   542
  Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   543
  Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   544
  IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   545
  Node* then = igvn.transform(new IfTrueNode(iff));
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   546
  Node* elsen = igvn.transform(new IfFalseNode(iff));
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   547
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   548
  Node* new_loadp = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   549
                                                                    (const TypePtr*) in_val->bottom_type(), MemNode::unordered, barrier->is_weak()));
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   550
50875
2217b2fc29ea 8205993: ZGC: Fix typos and incorrect indentations
pliden
parents: 50803
diff changeset
   551
  // Create the final region/phi pair to converge cntl/data paths to downstream code
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   552
  Node* result_region = igvn.transform(new RegionNode(3));
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   553
  result_region->set_req(1, then);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   554
  result_region->set_req(2, elsen);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   555
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   556
  Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM));
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   557
  result_phi->set_req(1, new_loadp);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   558
  result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop));
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   559
55536
8313c42345d5 8226287: Make process_users_of_allocation handle gc barriers
neliasso
parents: 55326
diff changeset
   560
8313c42345d5 8226287: Make process_users_of_allocation handle gc barriers
neliasso
parents: 55326
diff changeset
   561
  igvn.replace_node(out_ctrl, result_region);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   562
  igvn.replace_node(out_res, result_phi);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   563
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   564
  assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!");
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   565
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   566
  igvn.remove_dead_node(barrier);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   567
  igvn.remove_dead_node(out_ctrl);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   568
  igvn.remove_dead_node(out_res);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   569
51485
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   570
  assert(is_gc_barrier_node(result_phi), "sanity");
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   571
  assert(step_over_gc_barrier(result_phi) == in_val, "sanity");
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   572
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   573
  phase->C->print_method(PHASE_BARRIER_EXPANSION, 4, barrier->_idx);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   574
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   575
52711
f480ad035c73 8214172: GC interface entry points for loop opts
roland
parents: 52627
diff changeset
   576
bool ZBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   577
  ZBarrierSetC2State* s = state();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   578
  if (s->load_barrier_count() > 0) {
52829
af52abc1f61e 8214541: ZGC: Refactoring from JDK-8214172 may leave PhaseIterGVN::_delay_transform set
roland
parents: 52759
diff changeset
   579
    PhaseMacroExpand macro(igvn);
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   580
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   581
    int skipped = 0;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   582
    while (s->load_barrier_count() > skipped) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   583
      int load_barrier_count = s->load_barrier_count();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   584
      LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   585
      if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) {
50875
2217b2fc29ea 8205993: ZGC: Fix typos and incorrect indentations
pliden
parents: 50803
diff changeset
   586
        // Node is unreachable, so don't try to expand it
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   587
        s->remove_load_barrier_node(n);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   588
        continue;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   589
      }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   590
      if (!n->can_be_eliminated()) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   591
        skipped++;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   592
        continue;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   593
      }
52711
f480ad035c73 8214172: GC interface entry points for loop opts
roland
parents: 52627
diff changeset
   594
      expand_loadbarrier_node(&macro, n);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   595
      assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
53439
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
   596
      if (C->failing()) {
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
   597
        return true;
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
   598
      }
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   599
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   600
    while (s->load_barrier_count() > 0) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   601
      int load_barrier_count = s->load_barrier_count();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   602
      LoadBarrierNode* n = s->load_barrier_node(load_barrier_count - 1);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   603
      assert(!(igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())), "should have been processed already");
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   604
      assert(!n->can_be_eliminated(), "should have been processed already");
52711
f480ad035c73 8214172: GC interface entry points for loop opts
roland
parents: 52627
diff changeset
   605
      expand_loadbarrier_node(&macro, n);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   606
      assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
53439
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
   607
      if (C->failing()) {
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
   608
        return true;
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
   609
      }
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   610
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   611
    igvn.set_delay_transform(false);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   612
    igvn.optimize();
53439
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
   613
    if (C->failing()) {
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
   614
      return true;
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
   615
    }
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   616
  }
53439
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
   617
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   618
  return false;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   619
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   620
51485
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   621
Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const {
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   622
  Node* node = c;
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   623
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   624
  // 1. This step follows potential oop projections of a load barrier before expansion
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   625
  if (node->is_Proj()) {
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   626
    node = node->in(0);
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   627
  }
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   628
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   629
  // 2. This step checks for unexpanded load barriers
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   630
  if (node->is_LoadBarrier()) {
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   631
    return node->in(LoadBarrierNode::Oop);
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   632
  }
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   633
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   634
  // 3. This step checks for the phi corresponding to an optimized load barrier expansion
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   635
  if (node->is_Phi()) {
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   636
    PhiNode* phi = node->as_Phi();
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   637
    Node* n = phi->in(1);
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   638
    if (n != NULL && n->is_LoadBarrierSlowReg()) {
51485
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   639
      assert(c == node, "projections from step 1 should only be seen before macro expansion");
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   640
      return phi->in(2);
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   641
    }
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   642
  }
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   643
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   644
  return c;
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   645
}
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   646
55536
8313c42345d5 8226287: Make process_users_of_allocation handle gc barriers
neliasso
parents: 55326
diff changeset
   647
Node* ZBarrierSetC2::step_over_gc_barrier_ctrl(Node* c) const {
8313c42345d5 8226287: Make process_users_of_allocation handle gc barriers
neliasso
parents: 55326
diff changeset
   648
  Node* node = c;
8313c42345d5 8226287: Make process_users_of_allocation handle gc barriers
neliasso
parents: 55326
diff changeset
   649
8313c42345d5 8226287: Make process_users_of_allocation handle gc barriers
neliasso
parents: 55326
diff changeset
   650
  // 1. This step follows potential ctrl projections of a load barrier before expansion
8313c42345d5 8226287: Make process_users_of_allocation handle gc barriers
neliasso
parents: 55326
diff changeset
   651
  if (node->is_Proj()) {
8313c42345d5 8226287: Make process_users_of_allocation handle gc barriers
neliasso
parents: 55326
diff changeset
   652
    node = node->in(0);
8313c42345d5 8226287: Make process_users_of_allocation handle gc barriers
neliasso
parents: 55326
diff changeset
   653
  }
8313c42345d5 8226287: Make process_users_of_allocation handle gc barriers
neliasso
parents: 55326
diff changeset
   654
8313c42345d5 8226287: Make process_users_of_allocation handle gc barriers
neliasso
parents: 55326
diff changeset
   655
  // 2. This step checks for unexpanded load barriers
8313c42345d5 8226287: Make process_users_of_allocation handle gc barriers
neliasso
parents: 55326
diff changeset
   656
  if (node->is_LoadBarrier()) {
8313c42345d5 8226287: Make process_users_of_allocation handle gc barriers
neliasso
parents: 55326
diff changeset
   657
    return node->in(LoadBarrierNode::Control);
8313c42345d5 8226287: Make process_users_of_allocation handle gc barriers
neliasso
parents: 55326
diff changeset
   658
  }
8313c42345d5 8226287: Make process_users_of_allocation handle gc barriers
neliasso
parents: 55326
diff changeset
   659
8313c42345d5 8226287: Make process_users_of_allocation handle gc barriers
neliasso
parents: 55326
diff changeset
   660
  return c;
8313c42345d5 8226287: Make process_users_of_allocation handle gc barriers
neliasso
parents: 55326
diff changeset
   661
}
8313c42345d5 8226287: Make process_users_of_allocation handle gc barriers
neliasso
parents: 55326
diff changeset
   662
52036
5d6d636cefff 8211776: 8210887 broke arraycopy optimization when ZGC is enabled
roland
parents: 51489
diff changeset
   663
bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const {
5d6d636cefff 8211776: 8210887 broke arraycopy optimization when ZGC is enabled
roland
parents: 51489
diff changeset
   664
  return type == T_OBJECT || type == T_ARRAY;
5d6d636cefff 8211776: 8210887 broke arraycopy optimization when ZGC is enabled
roland
parents: 51489
diff changeset
   665
}
5d6d636cefff 8211776: 8210887 broke arraycopy optimization when ZGC is enabled
roland
parents: 51489
diff changeset
   666
52471
04d7e790aa2e 8213489: GC/C2 abstraction for Compile::final_graph_reshaping()
rkennke
parents: 52426
diff changeset
   667
bool ZBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const {
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   668
  switch (opcode) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   669
    case Op_LoadBarrier:
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   670
      assert(0, "There should be no load barriers left");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   671
    case Op_ZGetAndSetP:
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   672
    case Op_ZCompareAndExchangeP:
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   673
    case Op_ZCompareAndSwapP:
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   674
    case Op_ZWeakCompareAndSwapP:
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   675
    case Op_LoadBarrierSlowReg:
52471
04d7e790aa2e 8213489: GC/C2 abstraction for Compile::final_graph_reshaping()
rkennke
parents: 52426
diff changeset
   676
#ifdef ASSERT
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   677
      if (VerifyOptoOopOffsets) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   678
        MemNode *mem = n->as_Mem();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   679
        // Check to see if address types have grounded out somehow.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   680
        const TypeInstPtr *tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   681
        ciInstanceKlass *k = tp->klass()->as_instance_klass();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   682
        bool oop_offset_is_sane = k->contains_field_offset(tp->offset());
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   683
        assert(!tp || oop_offset_is_sane, "");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   684
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   685
#endif
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   686
      return true;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   687
    default:
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   688
      return false;
53439
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
   689
  }
52471
04d7e790aa2e 8213489: GC/C2 abstraction for Compile::final_graph_reshaping()
rkennke
parents: 52426
diff changeset
   690
}
04d7e790aa2e 8213489: GC/C2 abstraction for Compile::final_graph_reshaping()
rkennke
parents: 52426
diff changeset
   691
52627
e7d8ea5bfc8f 8213746: GC/C2 abstraction for C2 matcher
rkennke
parents: 52568
diff changeset
   692
bool ZBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const {
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   693
  switch(opcode) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   694
    case Op_CallLeaf:
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   695
      if (n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() ||
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   696
          n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   697
        mem_op = true;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   698
        mem_addr_idx = TypeFunc::Parms + 1;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   699
        return true;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   700
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   701
      return false;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   702
    default:
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   703
      return false;
52627
e7d8ea5bfc8f 8213746: GC/C2 abstraction for C2 matcher
rkennke
parents: 52568
diff changeset
   704
  }
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   705
}
53439
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
   706
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   707
bool ZBarrierSetC2::matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   708
  switch(opcode) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   709
    case Op_ZCompareAndExchangeP:
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   710
    case Op_ZCompareAndSwapP:
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   711
    case Op_ZWeakCompareAndSwapP: {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   712
      Node *mem = n->in(MemNode::Address);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   713
      Node *keepalive = n->in(5);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   714
      Node *pair1 = new BinaryNode(mem, keepalive);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   715
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   716
      Node *newval = n->in(MemNode::ValueIn);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   717
      Node *oldval = n->in(LoadStoreConditionalNode::ExpectedIn);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   718
      Node *pair2 = new BinaryNode(oldval, newval);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   719
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   720
      n->set_req(MemNode::Address, pair1);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   721
      n->set_req(MemNode::ValueIn, pair2);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   722
      n->del_req(5);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   723
      n->del_req(LoadStoreConditionalNode::ExpectedIn);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   724
      return true;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   725
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   726
    case Op_ZGetAndSetP: {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   727
      Node *keepalive = n->in(4);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   728
      Node *newval = n->in(MemNode::ValueIn);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   729
      Node *pair = new BinaryNode(newval, keepalive);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   730
      n->set_req(MemNode::ValueIn, pair);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   731
      n->del_req(4);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   732
      return true;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   733
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   734
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   735
    default:
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   736
      return false;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   737
  }
52627
e7d8ea5bfc8f 8213746: GC/C2 abstraction for C2 matcher
rkennke
parents: 52568
diff changeset
   738
}
e7d8ea5bfc8f 8213746: GC/C2 abstraction for C2 matcher
rkennke
parents: 52568
diff changeset
   739
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   740
// == Verification ==
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   741
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   742
#ifdef ASSERT
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   743
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   744
static bool look_for_barrier(Node* n, bool post_parse, VectorSet& visited) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   745
  if (visited.test_set(n->_idx)) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   746
    return true;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   747
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   748
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   749
  for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   750
    Node* u = n->fast_out(i);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   751
    if (u->is_LoadBarrier()) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   752
    } else if ((u->is_Phi() || u->is_CMove()) && !post_parse) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   753
      if (!look_for_barrier(u, post_parse, visited)) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   754
        return false;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   755
      }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   756
    } else if (u->Opcode() == Op_EncodeP || u->Opcode() == Op_DecodeN) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   757
      if (!look_for_barrier(u, post_parse, visited)) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   758
        return false;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   759
      }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   760
    } else if (u->Opcode() != Op_SCMemProj) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   761
      tty->print("bad use"); u->dump();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   762
      return false;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   763
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   764
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   765
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   766
  return true;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   767
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   768
52426
38bf0c9c4e64 8213384: Move G1/C2 barrier verification into G1BarrierSetC2
rkennke
parents: 52424
diff changeset
   769
void ZBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   770
  switch(phase) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   771
    case BarrierSetC2::BeforeOptimize:
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   772
    case BarrierSetC2::BeforeLateInsertion:
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   773
      assert(state()->load_barrier_count() == 0, "No barriers inserted yet");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   774
      break;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   775
    case BarrierSetC2::BeforeMacroExpand:
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   776
      // Barrier placement should be set by now.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   777
      verify_gc_barriers(false /*post_parse*/);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   778
      break;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   779
    case BarrierSetC2::BeforeCodeGen:
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   780
      // Barriers has been fully expanded.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   781
      assert(state()->load_barrier_count() == 0, "No more macro barriers");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   782
      break;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   783
    default:
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   784
      assert(0, "Phase without verification");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   785
  }
52426
38bf0c9c4e64 8213384: Move G1/C2 barrier verification into G1BarrierSetC2
rkennke
parents: 52424
diff changeset
   786
}
38bf0c9c4e64 8213384: Move G1/C2 barrier verification into G1BarrierSetC2
rkennke
parents: 52424
diff changeset
   787
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   788
// post_parse implies that there might be load barriers without uses after parsing
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   789
// That only applies when adding barriers at parse time.
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   790
void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   791
  ZBarrierSetC2State* s = state();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   792
  Compile* C = Compile::current();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   793
  ResourceMark rm;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   794
  VectorSet visited(Thread::current()->resource_area());
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   795
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   796
  for (int i = 0; i < s->load_barrier_count(); i++) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   797
    LoadBarrierNode* n = s->load_barrier_node(i);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   798
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   799
    // The dominating barrier on the same address if it exists and
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   800
    // this barrier must not be applied on the value from the same
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   801
    // load otherwise the value is not reloaded before it's used the
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   802
    // second time.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   803
    assert(n->in(LoadBarrierNode::Similar)->is_top() ||
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   804
           (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   805
            n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) &&
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   806
            n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)),
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   807
           "broken similar edge");
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   808
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   809
    assert(n->as_LoadBarrier()->has_true_uses(),
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   810
           "found unneeded load barrier");
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   811
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   812
    // Several load barrier nodes chained through their Similar edge
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   813
    // break the code that remove the barriers in final graph reshape.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   814
    assert(n->in(LoadBarrierNode::Similar)->is_top() ||
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   815
           (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   816
            n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Similar)->is_top()),
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   817
           "chain of Similar load barriers");
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   818
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   819
    if (!n->in(LoadBarrierNode::Similar)->is_top()) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   820
      ResourceMark rm;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   821
      Unique_Node_List wq;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   822
      Node* other = n->in(LoadBarrierNode::Similar)->in(0);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   823
      wq.push(n);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   824
      for (uint next = 0; next < wq.size(); ++next) {
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   825
        Node *nn = wq.at(next);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   826
        assert(nn->is_CFG(), "");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   827
        assert(!nn->is_SafePoint(), "");
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   828
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   829
        if (nn == other) {
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   830
          continue;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   831
        }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   832
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   833
        if (nn->is_Region()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   834
          for (uint i = 1; i < nn->req(); i++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   835
            Node* m = nn->in(i);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   836
            if (m != NULL) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   837
              wq.push(m);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   838
            }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   839
          }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   840
        } else {
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   841
          Node* m = nn->in(0);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   842
          if (m != NULL) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   843
            wq.push(m);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   844
          }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   845
        }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   846
      }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   847
    }
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   848
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   849
}
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   850
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   851
#endif // end verification code
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   852
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   853
static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   854
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   855
// This code is cloning all uses of a load that is between a call and the catch blocks,
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   856
// to each use.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   857
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   858
static bool fixup_uses_in_catch(PhaseIdealLoop *phase, Node *start_ctrl, Node *node) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   859
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   860
  if (!phase->has_ctrl(node)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   861
    // This node is floating - doesn't need to be cloned.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   862
    assert(node != start_ctrl, "check");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   863
    return false;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   864
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   865
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   866
  Node* ctrl = phase->get_ctrl(node);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   867
  if (ctrl != start_ctrl) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   868
    // We are in a successor block - the node is ok.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   869
    return false; // Unwind
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   870
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   871
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   872
  // Process successor nodes
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   873
  int outcnt = node->outcnt();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   874
  for (int i = 0; i < outcnt; i++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   875
    Node* n = node->raw_out(0);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   876
    assert(!n->is_LoadBarrier(), "Sanity");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   877
    // Calling recursively, visiting leafs first
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   878
    fixup_uses_in_catch(phase, start_ctrl, n);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   879
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   880
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   881
  // Now all successors are outside
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   882
  // - Clone this node to both successors
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   883
  int no_succs = node->outcnt();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   884
  assert(!node->is_Store(), "Stores not expected here");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   885
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   886
  // In some very rare cases a load that doesn't need a barrier will end up here
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   887
  // Treat it as a LoadP and the insertion of phis will be done correctly.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   888
  if (node->is_Load()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   889
    call_catch_cleanup_one(phase, node->as_Load(), phase->get_ctrl(node));
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   890
  } else {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   891
    for (DUIterator_Fast jmax, i = node->fast_outs(jmax); i < jmax; i++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   892
      Node* use = node->fast_out(i);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   893
      Node* clone = node->clone();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   894
      assert(clone->outcnt() == 0, "");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   895
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   896
      assert(use->find_edge(node) != -1, "check");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   897
      phase->igvn().rehash_node_delayed(use);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   898
      use->replace_edge(node, clone);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   899
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   900
      Node* new_ctrl;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   901
      if (use->is_block_start()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   902
        new_ctrl = use;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   903
      } else if (use->is_CFG()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   904
        new_ctrl = use->in(0);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   905
        assert (new_ctrl != NULL, "");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   906
      } else {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   907
        new_ctrl = phase->get_ctrl(use);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   908
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   909
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   910
      phase->set_ctrl(clone, new_ctrl);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   911
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   912
      if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr("  Clone op %i as %i to control %i", node->_idx, clone->_idx, new_ctrl->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   913
      phase->igvn().register_new_node_with_optimizer(clone);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   914
      --i, --jmax;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   915
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   916
    assert(node->outcnt() == 0, "must be empty now");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   917
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   918
    // Node node is dead.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   919
    phase->igvn().remove_dead_node(node);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   920
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   921
  return true; // unwind - return if a use was processed
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   922
}
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   923
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   924
// Clone a load to a specific catch_proj
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   925
static Node* clone_load_to_catchproj(PhaseIdealLoop* phase, Node* load, Node* catch_proj) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   926
  Node* cloned_load = load->clone();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   927
  cloned_load->set_req(0, catch_proj);      // set explicit control
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   928
  phase->set_ctrl(cloned_load, catch_proj); // update
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   929
  if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr("  Clone LOAD %i as %i to control %i", load->_idx, cloned_load->_idx, catch_proj->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   930
  phase->igvn().register_new_node_with_optimizer(cloned_load);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   931
  return cloned_load;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   932
}
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   933
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   934
static Node* get_dominating_region(PhaseIdealLoop* phase, Node* node, Node* stop) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   935
  Node* region = node;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   936
  while (!region->isa_Region()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   937
    Node *up = phase->idom(region);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   938
    assert(up != region, "Must not loop");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   939
    assert(up != stop,   "Must not find original control");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   940
    region = up;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   941
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   942
  return region;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   943
}
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   944
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   945
// Clone this load to each catch block
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   946
static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   947
  bool trace = phase->C->directive()->ZTraceLoadBarriersOption;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   948
  phase->igvn().set_delay_transform(true);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   949
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   950
  // Verify pre conditions
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   951
  assert(ctrl->isa_Proj() && ctrl->in(0)->isa_Call(), "Must be a call proj");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   952
  assert(ctrl->raw_out(0)->isa_Catch(), "Must be a catch");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   953
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   954
  if (ctrl->raw_out(0)->isa_Catch()->outcnt() == 1) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   955
    if (trace) tty->print_cr("Cleaning up catch: Skipping load %i, call with single catch", load->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   956
    return;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   957
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   958
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   959
  // Process the loads successor nodes - if any is between
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   960
  // the call and the catch blocks, they need to be cloned to.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   961
  // This is done recursively
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   962
  int outcnt = load->outcnt();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   963
  uint index = 0;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   964
  for (int i = 0; i < outcnt; i++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   965
    if (index < load->outcnt()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   966
      Node *n = load->raw_out(index);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   967
      assert(!n->is_LoadBarrier(), "Sanity");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   968
      if (!fixup_uses_in_catch(phase, ctrl, n)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   969
        // if no successor was cloned, progress to next out.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   970
        index++;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   971
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   972
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   973
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   974
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   975
  // Now all the loads uses has been cloned down
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   976
  // Only thing left is to clone the loads, but they must end up
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   977
  // first in the catch blocks.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   978
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   979
  // We clone the loads oo the catch blocks only when needed.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   980
  // An array is used to map the catch blocks to each lazily cloned load.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   981
  // In that way no extra unnecessary loads are cloned.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   982
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   983
  // Any use dominated by original block must have an phi and a region added
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   984
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   985
  Node* catch_node = ctrl->raw_out(0);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   986
  int number_of_catch_projs = catch_node->outcnt();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   987
  Node** proj_to_load_mapping = NEW_RESOURCE_ARRAY(Node*, number_of_catch_projs);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   988
  Copy::zero_to_bytes(proj_to_load_mapping, sizeof(Node*) * number_of_catch_projs);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   989
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   990
  // The phi_map is used to keep track of where phis have already been inserted
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   991
  int phi_map_len = phase->C->unique();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   992
  Node** phi_map = NEW_RESOURCE_ARRAY(Node*, phi_map_len);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   993
  Copy::zero_to_bytes(phi_map, sizeof(Node*) * phi_map_len);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   994
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   995
  for (unsigned int i = 0; i  < load->outcnt(); i++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   996
    Node* load_use_control = NULL;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   997
    Node* load_use = load->raw_out(i);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   998
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   999
    if (phase->has_ctrl(load_use)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1000
      load_use_control = phase->get_ctrl(load_use);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1001
    } else {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1002
      load_use_control = load_use->in(0);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1003
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1004
    assert(load_use_control != NULL, "sanity");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1005
    if (trace) tty->print_cr("  Handling use: %i, with control: %i", load_use->_idx, load_use_control->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1006
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1007
    // Some times the loads use is a phi. For them we need to determine from which catch block
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1008
    // the use is defined.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1009
    bool load_use_is_phi = false;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1010
    unsigned int load_use_phi_index = 0;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1011
    Node* phi_ctrl = NULL;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1012
    if (load_use->is_Phi()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1013
      // Find phi input that matches load
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1014
      for (unsigned int u = 1; u < load_use->req(); u++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1015
        if (load_use->in(u) == load) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1016
          load_use_is_phi = true;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1017
          load_use_phi_index = u;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1018
          assert(load_use->in(0)->is_Region(), "Region or broken");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1019
          phi_ctrl = load_use->in(0)->in(u);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1020
          assert(phi_ctrl->is_CFG(), "check");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1021
          assert(phi_ctrl != load,   "check");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1022
          break;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1023
        }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1024
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1025
      assert(load_use_is_phi,        "must find");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1026
      assert(load_use_phi_index > 0, "sanity");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1027
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1028
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1029
    // For each load use, see which catch projs dominates, create load clone lazily and reconnect
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1030
    bool found_dominating_catchproj = false;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1031
    for (int c = 0; c < number_of_catch_projs; c++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1032
      Node* catchproj = catch_node->raw_out(c);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1033
      assert(catchproj != NULL && catchproj->isa_CatchProj(), "Sanity");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1034
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1035
      if (!phase->is_dominator(catchproj, load_use_control)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1036
        if (load_use_is_phi && phase->is_dominator(catchproj, phi_ctrl)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1037
          // The loads use is local to the catchproj.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1038
          // fall out and replace load with catch-local load clone.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1039
        } else {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1040
          continue;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1041
        }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1042
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1043
      assert(!found_dominating_catchproj, "Max one should match");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1044
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1045
      // Clone loads to catch projs
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1046
      Node* load_clone = proj_to_load_mapping[c];
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1047
      if (load_clone == NULL) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1048
        load_clone = clone_load_to_catchproj(phase, load, catchproj);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1049
        proj_to_load_mapping[c] = load_clone;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1050
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1051
      phase->igvn().rehash_node_delayed(load_use);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1052
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1053
      if (load_use_is_phi) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1054
        // phis are special - the load is defined from a specific control flow
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1055
        load_use->set_req(load_use_phi_index, load_clone);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1056
      } else {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1057
        // Multipe edges can be replaced at once - on calls for example
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1058
        load_use->replace_edge(load, load_clone);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1059
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1060
      --i; // more than one edge can have been removed, but the next is in later iterations
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1061
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1062
      // We could break the for-loop after finding a dominating match.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1063
      // But keep iterating to catch any bad idom early.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1064
      found_dominating_catchproj = true;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1065
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1066
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1067
    // We found no single catchproj that dominated the use - The use is at a point after
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1068
    // where control flow from multiple catch projs have merged. We will have to create
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1069
    // phi nodes before the use and tie the output from the cloned loads together. It
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1070
    // can be a single phi or a number of chained phis, depending on control flow
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1071
    if (!found_dominating_catchproj) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1072
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1073
      // Use phi-control if use is a phi
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1074
      if (load_use_is_phi) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1075
        load_use_control = phi_ctrl;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1076
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1077
      assert(phase->is_dominator(ctrl, load_use_control), "Common use but no dominator");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1078
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1079
      // Clone a load on all paths
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1080
      for (int c = 0; c < number_of_catch_projs; c++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1081
        Node* catchproj = catch_node->raw_out(c);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1082
        Node* load_clone = proj_to_load_mapping[c];
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1083
        if (load_clone == NULL) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1084
          load_clone = clone_load_to_catchproj(phase, load, catchproj);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1085
          proj_to_load_mapping[c] = load_clone;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1086
        }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1087
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1088
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1089
      // Move up dominator tree from use until dom front is reached
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1090
      Node* next_region = get_dominating_region(phase, load_use_control, ctrl);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1091
      while (phase->idom(next_region) != catch_node) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1092
        next_region = phase->idom(next_region);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1093
        if (trace) tty->print_cr("Moving up idom to region ctrl %i", next_region->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1094
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1095
      assert(phase->is_dominator(catch_node, next_region), "Sanity");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1096
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1097
      // Create or reuse phi node that collect all cloned loads and feed it to the use.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1098
      Node* test_phi = phi_map[next_region->_idx];
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1099
      if ((test_phi != NULL) && test_phi->is_Phi()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1100
        // Reuse an already created phi
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1101
        if (trace) tty->print_cr("    Using cached Phi %i on load_use %i", test_phi->_idx, load_use->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1102
        phase->igvn().rehash_node_delayed(load_use);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1103
        load_use->replace_edge(load, test_phi);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1104
        // Now this use is done
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1105
      } else {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1106
        // Otherwise we need to create one or more phis
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1107
        PhiNode* next_phi = new PhiNode(next_region, load->type());
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1108
        phi_map[next_region->_idx] = next_phi; // cache new phi
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1109
        phase->igvn().rehash_node_delayed(load_use);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1110
        load_use->replace_edge(load, next_phi);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1111
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1112
        int dominators_of_region = 0;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1113
        do {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1114
          // New phi, connect to region and add all loads as in.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1115
          Node* region = next_region;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1116
          assert(region->isa_Region() && region->req() > 2, "Catch dead region nodes");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1117
          PhiNode* new_phi = next_phi;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1118
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1119
          if (trace) tty->print_cr("Created Phi %i on load %i with control %i", new_phi->_idx, load->_idx, region->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1120
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1121
          // Need to add all cloned loads to the phi, taking care that the right path is matched
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1122
          dominators_of_region = 0; // reset for new region
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1123
          for (unsigned int reg_i = 1; reg_i < region->req(); reg_i++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1124
            Node* region_pred = region->in(reg_i);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1125
            assert(region_pred->is_CFG(), "check");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1126
            bool pred_has_dominator = false;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1127
            for (int c = 0; c < number_of_catch_projs; c++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1128
              Node* catchproj = catch_node->raw_out(c);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1129
              if (phase->is_dominator(catchproj, region_pred)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1130
                new_phi->set_req(reg_i, proj_to_load_mapping[c]);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1131
                if (trace) tty->print_cr(" - Phi in(%i) set to load %i", reg_i, proj_to_load_mapping[c]->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1132
                pred_has_dominator = true;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1133
                dominators_of_region++;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1134
                break;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1135
              }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1136
            }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1137
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1138
            // Sometimes we need to chain several phis.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1139
            if (!pred_has_dominator) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1140
              assert(dominators_of_region <= 1, "More than one region can't require extra phi");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1141
              if (trace) tty->print_cr(" - Region %i pred %i not dominated by catch proj", region->_idx, region_pred->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1142
              // Continue search on on this region_pred
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1143
              // - walk up to next region
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1144
              // - create a new phi and connect to first new_phi
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1145
              next_region = get_dominating_region(phase, region_pred, ctrl);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1146
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1147
              // Lookup if there already is a phi, create a new otherwise
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1148
              Node* test_phi = phi_map[next_region->_idx];
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1149
              if ((test_phi != NULL) && test_phi->is_Phi()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1150
                next_phi = test_phi->isa_Phi();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1151
                dominators_of_region++; // record that a match was found and that we are done
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1152
                if (trace) tty->print_cr("    Using cached phi Phi %i on control %i", next_phi->_idx, next_region->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1153
              } else {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1154
                next_phi = new PhiNode(next_region, load->type());
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1155
                phi_map[next_region->_idx] = next_phi;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1156
              }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1157
              new_phi->set_req(reg_i, next_phi);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1158
            }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1159
          }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1160
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1161
          new_phi->set_req(0, region);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1162
          phase->igvn().register_new_node_with_optimizer(new_phi);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1163
          phase->set_ctrl(new_phi, region);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1164
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1165
          assert(dominators_of_region != 0, "Must have found one this iteration");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1166
        } while (dominators_of_region == 1);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1167
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1168
      --i;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1169
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1170
  } // end of loop over uses
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1171
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1172
  assert(load->outcnt() == 0, "All uses should be handled");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1173
  phase->igvn().remove_dead_node(load);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1174
  phase->C->print_method(PHASE_CALL_CATCH_CLEANUP, 4, load->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1175
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1176
  // Now we should be home
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1177
  phase->igvn().set_delay_transform(false);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1178
}
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1179
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1180
// Sort out the loads that are between a call ant its catch blocks
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1181
static void process_catch_cleanup_candidate(PhaseIdealLoop* phase, LoadNode* load) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1182
  bool trace = phase->C->directive()->ZTraceLoadBarriersOption;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1183
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1184
  Node* ctrl = phase->get_ctrl(load);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1185
  if (!ctrl->is_Proj() || (ctrl->in(0) == NULL) || !ctrl->in(0)->isa_Call()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1186
    return;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1187
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1188
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1189
  Node* catch_node = ctrl->isa_Proj()->raw_out(0);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1190
  if (catch_node->is_Catch()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1191
    if (catch_node->outcnt() > 1) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1192
      call_catch_cleanup_one(phase, load, ctrl);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1193
    } else {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1194
      if (trace) tty->print_cr("Call catch cleanup with only one catch: load %i ", load->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1195
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1196
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1197
}
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1198
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1199
void ZBarrierSetC2::barrier_insertion_phase(Compile* C, PhaseIterGVN& igvn) const {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1200
  PhaseIdealLoop::optimize(igvn, LoopOptsZBarrierInsertion);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1201
  if (C->failing())  return;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1202
}
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1203
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1204
bool ZBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1205
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1206
  if (mode == LoopOptsZBarrierInsertion) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1207
    // First make sure all loads between call and catch are moved to the catch block
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1208
    clean_catch_blocks(phase);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1209
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1210
    // Then expand barriers on all loads
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1211
    insert_load_barriers(phase);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1212
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1213
    // Handle all Unsafe that need barriers.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1214
    insert_barriers_on_unsafe(phase);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1215
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1216
    phase->C->clear_major_progress();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1217
    return true;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1218
  } else {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1219
    return false;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1220
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1221
}
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1222
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1223
static bool can_simplify_cas(LoadStoreNode* node) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1224
  if (node->isa_LoadStoreConditional()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1225
    Node *expected_in = node->as_LoadStoreConditional()->in(LoadStoreConditionalNode::ExpectedIn);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1226
    return (expected_in->get_ptr_type() == TypePtr::NULL_PTR);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1227
  } else {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1228
    return false;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1229
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1230
}
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1231
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1232
static void insert_barrier_before_unsafe(PhaseIdealLoop* phase, LoadStoreNode* old_node) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1233
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1234
  Compile *C = phase->C;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1235
  PhaseIterGVN &igvn = phase->igvn();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1236
  LoadStoreNode* zclone = NULL;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1237
  bool is_weak = false;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1238
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1239
  Node *in_ctrl = old_node->in(MemNode::Control);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1240
  Node *in_mem  = old_node->in(MemNode::Memory);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1241
  Node *in_adr  = old_node->in(MemNode::Address);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1242
  Node *in_val  = old_node->in(MemNode::ValueIn);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1243
  const TypePtr *adr_type = old_node->adr_type();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1244
  const TypePtr* load_type = TypeOopPtr::BOTTOM; // The type for the load we are adding
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1245
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1246
  switch (old_node->Opcode()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1247
    case Op_CompareAndExchangeP: {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1248
      zclone = new ZCompareAndExchangePNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn),
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1249
              adr_type, old_node->get_ptr_type(), ((CompareAndExchangeNode*)old_node)->order());
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1250
      load_type = old_node->bottom_type()->is_ptr();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1251
      break;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1252
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1253
    case Op_WeakCompareAndSwapP: {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1254
      if (can_simplify_cas(old_node)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1255
        break;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1256
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1257
      is_weak  = true;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1258
      zclone = new ZWeakCompareAndSwapPNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn),
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1259
              ((CompareAndSwapNode*)old_node)->order());
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1260
      adr_type = TypePtr::BOTTOM;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1261
      break;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1262
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1263
    case Op_CompareAndSwapP: {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1264
      if (can_simplify_cas(old_node)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1265
        break;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1266
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1267
      zclone = new ZCompareAndSwapPNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn),
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1268
              ((CompareAndSwapNode*)old_node)->order());
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1269
      adr_type = TypePtr::BOTTOM;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1270
      break;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1271
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1272
    case Op_GetAndSetP: {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1273
      zclone = new ZGetAndSetPNode(in_ctrl, in_mem, in_adr, in_val, old_node->adr_type(), old_node->get_ptr_type());
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1274
      load_type = old_node->bottom_type()->is_ptr();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1275
      break;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1276
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1277
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1278
  if (zclone != NULL) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1279
    igvn.register_new_node_with_optimizer(zclone, old_node);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1280
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1281
    // Make load
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1282
    LoadPNode *load = new LoadPNode(NULL, in_mem, in_adr, adr_type, load_type, MemNode::unordered,
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1283
                                    LoadNode::DependsOnlyOnTest);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1284
    load_set_expanded_barrier(load);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1285
    igvn.register_new_node_with_optimizer(load);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1286
    igvn.replace_node(old_node, zclone);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1287
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1288
    Node *barrier = new LoadBarrierNode(C, NULL, in_mem, load, in_adr, is_weak);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1289
    Node *barrier_val = new ProjNode(barrier, LoadBarrierNode::Oop);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1290
    Node *barrier_ctrl = new ProjNode(barrier, LoadBarrierNode::Control);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1291
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1292
    igvn.register_new_node_with_optimizer(barrier);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1293
    igvn.register_new_node_with_optimizer(barrier_val);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1294
    igvn.register_new_node_with_optimizer(barrier_ctrl);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1295
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1296
    // loop over all of in_ctrl usages and move to barrier_ctrl
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1297
    for (DUIterator_Last imin, i = in_ctrl->last_outs(imin); i >= imin; --i) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1298
      Node *use = in_ctrl->last_out(i);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1299
      uint l;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1300
      for (l = 0; use->in(l) != in_ctrl; l++) {}
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1301
      igvn.replace_input_of(use, l, barrier_ctrl);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1302
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1303
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1304
    load->set_req(MemNode::Control, in_ctrl);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1305
    barrier->set_req(LoadBarrierNode::Control, in_ctrl);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1306
    zclone->add_req(barrier_val); // add req as keep alive.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1308
    C->print_method(PHASE_ADD_UNSAFE_BARRIER, 4, zclone->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1309
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1310
}
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1311
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1312
void ZBarrierSetC2::insert_barriers_on_unsafe(PhaseIdealLoop* phase) const {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1313
  Compile *C = phase->C;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1314
  PhaseIterGVN &igvn = phase->igvn();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1315
  uint new_ids = C->unique();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1316
  VectorSet visited(Thread::current()->resource_area());
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1317
  GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1318
  nodeStack.push(C->root());
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1319
  visited.test_set(C->root()->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1320
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1321
  // Traverse all nodes, visit all unsafe ops that require a barrier
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1322
  while (nodeStack.length() > 0) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1323
    Node *n = nodeStack.pop();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1324
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1325
    bool is_old_node = (n->_idx < new_ids); // don't process nodes that were created during cleanup
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1326
    if (is_old_node) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1327
      if (n->is_LoadStore()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1328
        LoadStoreNode* lsn = n->as_LoadStore();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1329
        if (lsn->has_barrier()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1330
          BasicType bt = lsn->in(MemNode::Address)->bottom_type()->basic_type();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1331
          assert ((bt == T_OBJECT || bt == T_ARRAY), "Sanity test");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1332
          insert_barrier_before_unsafe(phase, lsn);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1333
        }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1334
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1335
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1336
    for (uint i = 0; i < n->len(); i++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1337
      if (n->in(i)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1338
        if (!visited.test_set(n->in(i)->_idx)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1339
          nodeStack.push(n->in(i));
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1340
        }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1341
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1342
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1343
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1344
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1345
  igvn.optimize();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1346
  C->print_method(PHASE_ADD_UNSAFE_BARRIER, 2);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1347
}
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1348
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1349
// The purpose of ZBarrierSetC2::clean_catch_blocks is to prepare the IR for
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1350
// splicing in load barrier nodes.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1351
//
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1352
// The problem is that we might have instructions between a call and its catch nodes.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1353
// (This is usually handled in PhaseCFG:call_catch_cleanup, which clones mach nodes in
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1354
// already scheduled blocks.) We can't have loads that require barriers there,
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1355
// because we need to splice in new control flow, and that would violate the IR.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1356
//
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1357
// clean_catch_blocks find all Loads that require a barrier and clone them and any
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1358
// dependent instructions to each use. The loads must be in the beginning of the catch block
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1359
// before any store.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1360
//
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1361
// Sometimes the loads use will be at a place dominated by all catch blocks, then we need
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1362
// a load in each catch block, and a Phi at the dominated use.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1363
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1364
void ZBarrierSetC2::clean_catch_blocks(PhaseIdealLoop* phase) const {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1365
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1366
  Compile *C = phase->C;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1367
  uint new_ids = C->unique();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1368
  PhaseIterGVN &igvn = phase->igvn();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1369
  VectorSet visited(Thread::current()->resource_area());
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1370
  GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1371
  nodeStack.push(C->root());
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1372
  visited.test_set(C->root()->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1373
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1374
  // Traverse all nodes, visit all loads that require a barrier
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1375
  while(nodeStack.length() > 0) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1376
    Node *n = nodeStack.pop();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1377
55321
ddda023e6f66 8225509: clean_catch_blocks must add preds first
neliasso
parents: 55307
diff changeset
  1378
    for (uint i = 0; i < n->len(); i++) {
ddda023e6f66 8225509: clean_catch_blocks must add preds first
neliasso
parents: 55307
diff changeset
  1379
      if (n->in(i)) {
ddda023e6f66 8225509: clean_catch_blocks must add preds first
neliasso
parents: 55307
diff changeset
  1380
        if (!visited.test_set(n->in(i)->_idx)) {
ddda023e6f66 8225509: clean_catch_blocks must add preds first
neliasso
parents: 55307
diff changeset
  1381
          nodeStack.push(n->in(i));
ddda023e6f66 8225509: clean_catch_blocks must add preds first
neliasso
parents: 55307
diff changeset
  1382
        }
ddda023e6f66 8225509: clean_catch_blocks must add preds first
neliasso
parents: 55307
diff changeset
  1383
      }
ddda023e6f66 8225509: clean_catch_blocks must add preds first
neliasso
parents: 55307
diff changeset
  1384
    }
ddda023e6f66 8225509: clean_catch_blocks must add preds first
neliasso
parents: 55307
diff changeset
  1385
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1386
    bool is_old_node = (n->_idx < new_ids); // don't process nodes that were created during cleanup
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1387
    if (n->is_Load() && is_old_node) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1388
      LoadNode* load = n->isa_Load();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1389
      // only care about loads that will have a barrier
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1390
      if (load_require_barrier(load)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1391
        process_catch_cleanup_candidate(phase, load);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1392
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1393
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1394
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1395
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1396
  C->print_method(PHASE_CALL_CATCH_CLEANUP, 2);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1397
}
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1398
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1399
class DomDepthCompareClosure : public CompareClosure<LoadNode*> {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1400
  PhaseIdealLoop* _phase;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1401
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1402
public:
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1403
  DomDepthCompareClosure(PhaseIdealLoop* phase) : _phase(phase) { }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1404
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1405
  int do_compare(LoadNode* const &n1, LoadNode* const &n2) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1406
    int d1 = _phase->dom_depth(_phase->get_ctrl(n1));
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1407
    int d2 = _phase->dom_depth(_phase->get_ctrl(n2));
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1408
    if (d1 == d2) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1409
      // Compare index if the depth is the same, ensures all entries are unique.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1410
      return n1->_idx - n2->_idx;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1411
    } else {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1412
      return d2 - d1;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1413
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1414
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1415
};
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1416
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1417
// Traverse graph and add all loadPs to list, sorted by dom depth
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1418
void gather_loadnodes_sorted(PhaseIdealLoop* phase, GrowableArray<LoadNode*>* loadList) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1419
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1420
  VectorSet visited(Thread::current()->resource_area());
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1421
  GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1422
  DomDepthCompareClosure ddcc(phase);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1423
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1424
  nodeStack.push(phase->C->root());
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1425
  while(nodeStack.length() > 0) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1426
    Node *n = nodeStack.pop();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1427
    if (visited.test(n->_idx)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1428
      continue;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1429
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1430
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1431
    if (n->isa_Load()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1432
      LoadNode *load = n->as_Load();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1433
      if (load_require_barrier(load)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1434
        assert(phase->get_ctrl(load) != NULL, "sanity");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1435
        assert(phase->dom_depth(phase->get_ctrl(load)) != 0, "sanity");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1436
        loadList->insert_sorted(&ddcc, load);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1437
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1438
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1439
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1440
    visited.set(n->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1441
    for (uint i = 0; i < n->req(); i++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1442
      if (n->in(i)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1443
        if (!visited.test(n->in(i)->_idx)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1444
          nodeStack.push(n->in(i));
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
  1445
        }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
  1446
      }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
  1447
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
  1448
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
  1449
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
  1450
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1451
// Add LoadBarriers to all LoadPs
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1452
void ZBarrierSetC2::insert_load_barriers(PhaseIdealLoop* phase) const {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1453
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1454
  bool trace = phase->C->directive()->ZTraceLoadBarriersOption;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1455
  GrowableArray<LoadNode *> loadList(Thread::current()->resource_area(), 0, 0, NULL);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1456
  gather_loadnodes_sorted(phase, &loadList);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1457
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1458
  PhaseIterGVN &igvn = phase->igvn();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1459
  int count = 0;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1460
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1461
  for (GrowableArrayIterator<LoadNode *> loadIter = loadList.begin(); loadIter != loadList.end(); ++loadIter) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1462
    LoadNode *load = *loadIter;
52568
40474b7105f4 8213615: GC/C2 abstraction for escape analysis
rkennke
parents: 52471
diff changeset
  1463
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1464
    if (load_has_expanded_barrier(load)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1465
      continue;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1466
    }
53439
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
  1467
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1468
    do {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1469
      // Insert a barrier on a loadP
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1470
      // if another load is found that needs to be expanded first, retry on that one
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1471
      LoadNode* result = insert_one_loadbarrier(phase, load, phase->get_ctrl(load));
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1472
      while (result != NULL) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1473
        result = insert_one_loadbarrier(phase, result, phase->get_ctrl(result));
52568
40474b7105f4 8213615: GC/C2 abstraction for escape analysis
rkennke
parents: 52471
diff changeset
  1474
      }
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1475
    } while (!load_has_expanded_barrier(load));
52568
40474b7105f4 8213615: GC/C2 abstraction for escape analysis
rkennke
parents: 52471
diff changeset
  1476
  }
53439
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
  1477
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1478
  phase->C->print_method(PHASE_INSERT_BARRIER, 2);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1479
}
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1480
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1481
void push_antidependent_stores(PhaseIdealLoop* phase, Node_Stack& nodestack, LoadNode* start_load) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1482
  // push all stores on the same mem, that can_alias
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1483
  // Any load found must be handled first
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1484
  PhaseIterGVN &igvn = phase->igvn();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1485
  int load_alias_idx = igvn.C->get_alias_index(start_load->adr_type());
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1486
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1487
  Node *mem = start_load->in(1);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1488
  for (DUIterator_Fast imax, u = mem->fast_outs(imax); u < imax; u++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1489
    Node *mem_use = mem->fast_out(u);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1490
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1491
    if (mem_use == start_load) continue;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1492
    if (!mem_use->is_Store()) continue;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1493
    if (!phase->has_ctrl(mem_use)) continue;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1494
    if (phase->get_ctrl(mem_use) != phase->get_ctrl(start_load)) continue;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1495
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1496
    // add any aliasing store in this block
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1497
    StoreNode *store = mem_use->isa_Store();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1498
    const TypePtr *adr_type = store->adr_type();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1499
    if (igvn.C->can_alias(adr_type, load_alias_idx)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1500
      nodestack.push(store, 0);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1501
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1502
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1503
}
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1504
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1505
LoadNode* ZBarrierSetC2::insert_one_loadbarrier(PhaseIdealLoop* phase, LoadNode* start_load, Node* ctrl) const {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1506
  bool trace = phase->C->directive()->ZTraceLoadBarriersOption;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1507
  PhaseIterGVN &igvn = phase->igvn();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1508
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1509
  // Check for other loadPs at the same loop depth that is reachable by a DFS
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1510
  // - if found - return it. It needs to be inserted first
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1511
  // - otherwise proceed and insert barrier
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1512
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1513
  VectorSet visited(Thread::current()->resource_area());
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1514
  Node_Stack nodestack(100);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1515
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1516
  nodestack.push(start_load, 0);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1517
  push_antidependent_stores(phase, nodestack, start_load);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1518
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1519
  while(!nodestack.is_empty()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1520
    Node* n = nodestack.node(); // peek
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1521
    nodestack.pop();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1522
    if (visited.test(n->_idx)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1523
      continue;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1524
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1525
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1526
    if (n->is_Load() && n != start_load && load_require_barrier(n->as_Load()) && !load_has_expanded_barrier(n->as_Load())) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1527
      // Found another load that needs a barrier in the same block. Must expand later loads first.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1528
      if (trace) tty->print_cr(" * Found LoadP %i on DFS", n->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1529
      return n->as_Load(); // return node that should be expanded first
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1530
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1531
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1532
    if (!phase->has_ctrl(n)) continue;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1533
    if (phase->get_ctrl(n) != phase->get_ctrl(start_load)) continue;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1534
    if (n->is_Phi()) continue;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1535
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1536
    visited.set(n->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1537
    // push all children
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1538
    for (DUIterator_Fast imax, ii = n->fast_outs(imax); ii < imax; ii++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1539
      Node* c = n->fast_out(ii);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1540
      if (c != NULL) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1541
        nodestack.push(c, 0);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1542
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1543
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1544
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1545
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1546
  insert_one_loadbarrier_inner(phase, start_load, ctrl, visited);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1547
  return NULL;
52568
40474b7105f4 8213615: GC/C2 abstraction for escape analysis
rkennke
parents: 52471
diff changeset
  1548
}
40474b7105f4 8213615: GC/C2 abstraction for escape analysis
rkennke
parents: 52471
diff changeset
  1549
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1550
void ZBarrierSetC2::insert_one_loadbarrier_inner(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl, VectorSet visited2) const {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1551
  PhaseIterGVN &igvn = phase->igvn();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1552
  Compile* C = igvn.C;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1553
  bool trace = C->directive()->ZTraceLoadBarriersOption;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1554
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1555
  // create barrier
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1556
  Node* barrier = new LoadBarrierNode(C, NULL, load->in(LoadNode::Memory), NULL, load->in(LoadNode::Address), load_has_weak_barrier(load));
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1557
  Node* barrier_val = new ProjNode(barrier, LoadBarrierNode::Oop);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1558
  Node* barrier_ctrl = new ProjNode(barrier, LoadBarrierNode::Control);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1559
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1560
  if (trace) tty->print_cr("Insert load %i with barrier: %i and ctrl : %i", load->_idx, barrier->_idx, ctrl->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1561
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1562
  // Splice control
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1563
  // - insert barrier control diamond between loads ctrl and ctrl successor on path to block end.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1564
  // - If control successor is a catch, step over to next.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1565
  Node* ctrl_succ = NULL;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1566
  for (DUIterator_Fast imax, j = ctrl->fast_outs(imax); j < imax; j++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1567
    Node* tmp = ctrl->fast_out(j);
53439
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
  1568
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1569
    // - CFG nodes is the ones we are going to splice (1 only!)
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1570
    // - Phi nodes will continue to hang from the region node!
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1571
    // - self loops should be skipped
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1572
    if (tmp->is_Phi() || tmp == ctrl) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1573
      continue;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1574
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1575
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1576
    if (tmp->is_CFG()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1577
      assert(ctrl_succ == NULL, "There can be only one");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1578
      ctrl_succ = tmp;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1579
      continue;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1580
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1581
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1582
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1583
  // Now splice control
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1584
  assert(ctrl_succ != load, "sanity");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1585
  assert(ctrl_succ != NULL, "Broken IR");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1586
  bool found = false;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1587
  for(uint k = 0; k < ctrl_succ->req(); k++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1588
    if (ctrl_succ->in(k) == ctrl) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1589
      assert(!found, "sanity");
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1590
      if (trace) tty->print_cr(" Move CFG ctrl_succ %i to barrier_ctrl", ctrl_succ->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1591
      igvn.replace_input_of(ctrl_succ, k, barrier_ctrl);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1592
      found = true;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1593
      k--;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1594
    }
52568
40474b7105f4 8213615: GC/C2 abstraction for escape analysis
rkennke
parents: 52471
diff changeset
  1595
  }
53439
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
  1596
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1597
  // For all successors of ctrl - move all visited to become successors of barrier_ctrl instead
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1598
  for (DUIterator_Fast imax, r = ctrl->fast_outs(imax); r < imax; r++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1599
    Node* tmp = ctrl->fast_out(r);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1600
    if (visited2.test(tmp->_idx) && (tmp != load)) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1601
      if (trace) tty->print_cr(" Move ctrl_succ %i to barrier_ctrl", tmp->_idx);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1602
      igvn.replace_input_of(tmp, 0, barrier_ctrl);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1603
      --r; --imax;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1604
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1605
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1606
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1607
  // Move the loads user to the barrier
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1608
  for (DUIterator_Fast imax, i = load->fast_outs(imax); i < imax; i++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1609
    Node* u = load->fast_out(i);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1610
    if (u->isa_LoadBarrier()) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1611
      continue;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1612
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1613
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1614
    // find correct input  - replace with iterator?
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1615
    for(uint j = 0; j < u->req(); j++) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1616
      if (u->in(j) == load) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1617
        igvn.replace_input_of(u, j, barrier_val);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1618
        --i; --imax; // Adjust the iterator of the *outer* loop
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1619
        break; // some nodes (calls) might have several uses from the same node
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1620
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1621
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1622
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1623
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1624
  // Connect barrier to load and control
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1625
  barrier->set_req(LoadBarrierNode::Oop, load);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1626
  barrier->set_req(LoadBarrierNode::Control, ctrl);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1627
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1628
  igvn.rehash_node_delayed(load);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1629
  igvn.register_new_node_with_optimizer(barrier);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1630
  igvn.register_new_node_with_optimizer(barrier_val);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1631
  igvn.register_new_node_with_optimizer(barrier_ctrl);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1632
  load_set_expanded_barrier(load);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1633
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1634
  C->print_method(PHASE_INSERT_BARRIER, 3, load->_idx);
52711
f480ad035c73 8214172: GC interface entry points for loop opts
roland
parents: 52627
diff changeset
  1635
}
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1636
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1637
// The bad_mask in the ThreadLocalData shouldn't have an anti-dep-check.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1638
// The bad_mask address if of type TypeRawPtr, but that will alias
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1639
// InitializeNodes until the type system is expanded.
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1640
bool ZBarrierSetC2::needs_anti_dependence_check(const Node* node) const {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1641
  MachNode* mnode = node->as_Mach();
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1642
  if (mnode != NULL) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1643
    intptr_t offset = 0;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1644
    const TypePtr *adr_type2 = NULL;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1645
    const Node* base = mnode->get_base_and_disp(offset, adr_type2);
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1646
    if ((base != NULL) &&
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1647
        (base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_ThreadLocal) &&
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1648
        (offset == in_bytes(ZThreadLocalData::address_bad_mask_offset()))) {
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1649
      return false;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1650
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1651
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1652
  return true;
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
  1653
}