src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp
author neliasso
Thu, 31 Oct 2019 17:16:36 +0100
changeset 58931 304c63b17b07
parent 58516 d376d86b0a01
child 58958 7bfe7df764a2
permissions -rw-r--r--
8232896: ZGC: Enable C2 clone intrinsic Reviewed-by: pliden, kvn
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     1
/*
53223
df6cbf676c70 8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents: 52829
diff changeset
     2
 * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     4
 *
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     5
 * This code is free software; you can redistribute it and/or modify it
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     6
 * under the terms of the GNU General Public License version 2 only, as
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     7
 * published by the Free Software Foundation.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     8
 *
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     9
 * This code is distributed in the hope that it will be useful, but WITHOUT
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    12
 * version 2 for more details (a copy is included in the LICENSE file that
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    13
 * accompanied this code).
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    14
 *
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    15
 * You should have received a copy of the GNU General Public License version
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    16
 * 2 along with this work; if not, write to the Free Software Foundation,
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    18
 *
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    20
 * or visit www.oracle.com if you need additional information or have any
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    21
 * questions.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    22
 */
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    23
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    24
#include "precompiled.hpp"
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    25
#include "classfile/javaClasses.hpp"
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    26
#include "gc/z/c2/zBarrierSetC2.hpp"
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    27
#include "gc/z/zBarrierSet.hpp"
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    28
#include "gc/z/zBarrierSetAssembler.hpp"
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    29
#include "gc/z/zBarrierSetRuntime.hpp"
58931
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
    30
#include "opto/arraycopynode.hpp"
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    31
#include "opto/block.hpp"
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    32
#include "opto/compile.hpp"
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    33
#include "opto/graphKit.hpp"
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
    34
#include "opto/machnode.hpp"
58931
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
    35
#include "opto/macro.hpp"
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
    36
#include "opto/memnode.hpp"
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    37
#include "opto/node.hpp"
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    38
#include "opto/regalloc.hpp"
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
    39
#include "opto/rootnode.hpp"
58931
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
    40
#include "opto/runtime.hpp"
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
    41
#include "utilities/growableArray.hpp"
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    42
#include "utilities/macros.hpp"
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    43
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    44
class ZBarrierSetC2State : public ResourceObj {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    45
private:
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    46
  GrowableArray<ZLoadBarrierStubC2*>* _stubs;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    47
  Node_Array                          _live;
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    48
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    49
public:
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    50
  ZBarrierSetC2State(Arena* arena) :
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    51
    _stubs(new (arena) GrowableArray<ZLoadBarrierStubC2*>(arena, 8,  0, NULL)),
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    52
    _live(arena) {}
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    53
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    54
  GrowableArray<ZLoadBarrierStubC2*>* stubs() {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    55
    return _stubs;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    56
  }
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    57
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    58
  RegMask* live(const Node* node) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    59
    if (!node->is_Mach()) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    60
      // Don't need liveness for non-MachNodes
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    61
      return NULL;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    62
    }
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    63
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    64
    const MachNode* const mach = node->as_Mach();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    65
    if (mach->barrier_data() != ZLoadBarrierStrong &&
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    66
        mach->barrier_data() != ZLoadBarrierWeak) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    67
      // Don't need liveness data for nodes without barriers
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    68
      return NULL;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    69
    }
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    70
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    71
    RegMask* live = (RegMask*)_live[node->_idx];
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    72
    if (live == NULL) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    73
      live = new (Compile::current()->comp_arena()->Amalloc_D(sizeof(RegMask))) RegMask();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    74
      _live.map(node->_idx, (Node*)live);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    75
    }
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    76
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    77
    return live;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    78
  }
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    79
};
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    80
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    81
static ZBarrierSetC2State* barrier_set_state() {
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    82
  return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    83
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    84
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    85
ZLoadBarrierStubC2* ZLoadBarrierStubC2::create(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    86
  ZLoadBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2(node, ref_addr, ref, tmp, weak);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    87
  if (!Compile::current()->in_scratch_emit_size()) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    88
    barrier_set_state()->stubs()->append(stub);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    89
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    90
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    91
  return stub;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    92
}
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    93
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    94
ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) :
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    95
    _node(node),
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    96
    _ref_addr(ref_addr),
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    97
    _ref(ref),
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    98
    _tmp(tmp),
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
    99
    _weak(weak),
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   100
    _entry(),
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   101
    _continuation() {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   102
  assert_different_registers(ref, ref_addr.base());
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   103
  assert_different_registers(ref, ref_addr.index());
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   104
}
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   105
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   106
Address ZLoadBarrierStubC2::ref_addr() const {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   107
  return _ref_addr;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   108
}
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   109
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   110
Register ZLoadBarrierStubC2::ref() const {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   111
  return _ref;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   112
}
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   113
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   114
Register ZLoadBarrierStubC2::tmp() const {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   115
  return _tmp;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   116
}
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   117
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   118
address ZLoadBarrierStubC2::slow_path() const {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   119
  const DecoratorSet decorators = _weak ? ON_WEAK_OOP_REF : ON_STRONG_OOP_REF;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   120
  return ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   121
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   122
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   123
RegMask& ZLoadBarrierStubC2::live() const {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   124
  return *barrier_set_state()->live(_node);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   125
}
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   126
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   127
Label* ZLoadBarrierStubC2::entry() {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   128
  // The _entry will never be bound when in_scratch_emit_size() is true.
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   129
  // However, we still need to return a label that is not bound now, but
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   130
  // will eventually be bound. Any lable will do, as it will only act as
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   131
  // a placeholder, so we return the _continuation label.
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   132
  return Compile::current()->in_scratch_emit_size() ? &_continuation : &_entry;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   133
}
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   134
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   135
Label* ZLoadBarrierStubC2::continuation() {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   136
  return &_continuation;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   137
}
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   138
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   139
void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   140
  return new (comp_arena) ZBarrierSetC2State(comp_arena);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   141
}
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   142
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   143
void ZBarrierSetC2::late_barrier_analysis() const {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   144
  analyze_dominating_barriers();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   145
  compute_liveness_at_stubs();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   146
}
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   147
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   148
void ZBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   149
  MacroAssembler masm(&cb);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   150
  GrowableArray<ZLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   151
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   152
  for (int i = 0; i < stubs->length(); i++) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   153
    // Make sure there is enough space in the code buffer
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   154
    if (cb.insts()->maybe_expand_to_ensure_remaining(Compile::MAX_inst_size) && cb.blob() == NULL) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   155
      ciEnv::current()->record_failure("CodeCache is full");
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   156
      return;
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   157
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   158
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   159
    ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i));
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   160
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   161
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   162
  masm.flush();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   163
}
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   164
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   165
int ZBarrierSetC2::estimate_stub_size() const {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   166
  Compile* const C = Compile::current();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   167
  BufferBlob* const blob = C->scratch_buffer_blob();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   168
  GrowableArray<ZLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   169
  int size = 0;
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   170
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   171
  for (int i = 0; i < stubs->length(); i++) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   172
    CodeBuffer cb(blob->content_begin(), (address)C->scratch_locs_memory() - blob->content_begin());
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   173
    MacroAssembler masm(&cb);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   174
    ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i));
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   175
    size += cb.insts_size();
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   176
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   177
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   178
  return size;
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   179
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   180
52424
e3d79743f57d 8212243: More gc interface tweaks for arraycopy
roland
parents: 52224
diff changeset
   181
static bool barrier_needed(C2Access& access) {
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   182
  return ZBarrierSet::barrier_needed(access.decorators(), access.type());
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   183
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   184
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   185
Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   186
  Node* result = BarrierSetC2::load_at_resolved(access, val_type);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   187
  if (barrier_needed(access) && access.raw_access()->is_Mem()) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   188
    if ((access.decorators() & ON_WEAK_OOP_REF) != 0) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   189
      access.raw_access()->as_Load()->set_barrier_data(ZLoadBarrierWeak);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   190
    } else {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   191
      access.raw_access()->as_Load()->set_barrier_data(ZLoadBarrierStrong);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   192
    }
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   193
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   194
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   195
  return result;
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   196
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   197
52424
e3d79743f57d 8212243: More gc interface tweaks for arraycopy
roland
parents: 52224
diff changeset
   198
Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   199
                                                    Node* new_val, const Type* val_type) const {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   200
  Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   201
  if (barrier_needed(access)) {
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   202
    access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   203
  }
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   204
  return result;
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   205
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   206
52424
e3d79743f57d 8212243: More gc interface tweaks for arraycopy
roland
parents: 52224
diff changeset
   207
Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   208
                                                     Node* new_val, const Type* value_type) const {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   209
  Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   210
  if (barrier_needed(access)) {
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   211
    access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   212
  }
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   213
  return result;
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   214
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   215
52424
e3d79743f57d 8212243: More gc interface tweaks for arraycopy
roland
parents: 52224
diff changeset
   216
Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const {
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   217
  Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   218
  if (barrier_needed(access)) {
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   219
    access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   220
  }
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   221
  return result;
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   222
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   223
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   224
bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type,
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   225
                                                    bool is_clone, ArrayCopyPhase phase) const {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   226
  return type == T_OBJECT || type == T_ARRAY;
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   227
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   228
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   229
// == Dominating barrier elision ==
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   230
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   231
static bool block_has_safepoint(const Block* block, uint from, uint to) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   232
  for (uint i = from; i < to; i++) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   233
    if (block->get_node(i)->is_MachSafePoint()) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   234
      // Safepoint found
53439
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
   235
      return true;
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
   236
    }
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   237
  }
53439
4b42f828e0c3 8217257: ZGC: Minor cleanup of ZBarrierSetC2
pliden
parents: 53438
diff changeset
   238
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   239
  // Safepoint not found
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   240
  return false;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   241
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   242
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   243
static bool block_has_safepoint(const Block* block) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   244
  return block_has_safepoint(block, 0, block->number_of_nodes());
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   245
}
51485
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   246
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   247
static uint block_index(const Block* block, const Node* node) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   248
  for (uint j = 0; j < block->number_of_nodes(); ++j) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   249
    if (block->get_node(j) == node) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   250
      return j;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   251
    }
51485
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   252
  }
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   253
  ShouldNotReachHere();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   254
  return 0;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   255
}
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   256
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   257
void ZBarrierSetC2::analyze_dominating_barriers() const {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   258
  ResourceMark rm;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   259
  Compile* const C = Compile::current();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   260
  PhaseCFG* const cfg = C->cfg();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   261
  Block_List worklist;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   262
  Node_List mem_ops;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   263
  Node_List barrier_loads;
51485
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   264
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   265
  // Step 1 - Find accesses, and track them in lists
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   266
  for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   267
    const Block* const block = cfg->get_block(i);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   268
    for (uint j = 0; j < block->number_of_nodes(); ++j) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   269
      const Node* const node = block->get_node(j);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   270
      if (!node->is_Mach()) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   271
        continue;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   272
      }
51485
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   273
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   274
      MachNode* const mach = node->as_Mach();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   275
      switch (mach->ideal_Opcode()) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   276
      case Op_LoadP:
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   277
      case Op_CompareAndExchangeP:
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   278
      case Op_CompareAndSwapP:
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   279
      case Op_GetAndSetP:
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   280
        if (mach->barrier_data() == ZLoadBarrierStrong) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   281
          barrier_loads.push(mach);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   282
        }
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   283
      case Op_StoreP:
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   284
        mem_ops.push(mach);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   285
        break;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   286
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   287
      default:
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   288
        break;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   289
      }
51485
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   290
    }
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   291
  }
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 50875
diff changeset
   292
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   293
  // Step 2 - Find dominating accesses for each load
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   294
  for (uint i = 0; i < barrier_loads.size(); i++) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   295
    MachNode* const load = barrier_loads.at(i)->as_Mach();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   296
    const TypePtr* load_adr_type = NULL;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   297
    intptr_t load_offset = 0;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   298
    const Node* const load_obj = load->get_base_and_disp(load_offset, load_adr_type);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   299
    Block* const load_block = cfg->get_block_for_node(load);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   300
    const uint load_index = block_index(load_block, load);
52471
04d7e790aa2e 8213489: GC/C2 abstraction for Compile::final_graph_reshaping()
rkennke
parents: 52426
diff changeset
   301
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   302
    for (uint j = 0; j < mem_ops.size(); j++) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   303
      MachNode* mem = mem_ops.at(j)->as_Mach();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   304
      const TypePtr* mem_adr_type = NULL;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   305
      intptr_t mem_offset = 0;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   306
      const Node* mem_obj = mem_obj = mem->get_base_and_disp(mem_offset, mem_adr_type);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   307
      Block* mem_block = cfg->get_block_for_node(mem);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   308
      uint mem_index = block_index(mem_block, mem);
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   309
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   310
      if (load_obj == NodeSentinel || mem_obj == NodeSentinel ||
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   311
          load_obj == NULL || mem_obj == NULL ||
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   312
          load_offset < 0 || mem_offset < 0) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   313
        continue;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   314
      }
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   315
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   316
      if (mem_obj != load_obj || mem_offset != load_offset) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   317
        // Not the same addresses, not a candidate
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   318
        continue;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   319
      }
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   320
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   321
      if (load_block == mem_block) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   322
        // Earlier accesses in the same block
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   323
        if (mem_index < load_index && !block_has_safepoint(mem_block, mem_index + 1, load_index)) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   324
          load->set_barrier_data(ZLoadBarrierElided);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   325
        }
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   326
      } else if (mem_block->dominates(load_block)) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   327
        // Dominating block? Look around for safepoints
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   328
        ResourceMark rm;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   329
        Block_List stack;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   330
        VectorSet visited(Thread::current()->resource_area());
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   331
        stack.push(load_block);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   332
        bool safepoint_found = block_has_safepoint(load_block);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   333
        while (!safepoint_found && stack.size() > 0) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   334
          Block* block = stack.pop();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   335
          if (visited.test_set(block->_pre_order)) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   336
            continue;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   337
          }
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   338
          if (block_has_safepoint(block)) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   339
            safepoint_found = true;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   340
            break;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   341
          }
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   342
          if (block == mem_block) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   343
            continue;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   344
          }
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   345
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   346
          // Push predecessor blocks
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   347
          for (uint p = 1; p < block->num_preds(); ++p) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   348
            Block* pred = cfg->get_block_for_node(block->pred(p));
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   349
            stack.push(pred);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   350
          }
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   351
        }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   352
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   353
        if (!safepoint_found) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   354
          load->set_barrier_data(ZLoadBarrierElided);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   355
        }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   356
      }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   357
    }
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   358
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   359
}
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   360
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   361
// == Reduced spilling optimization ==
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   362
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   363
void ZBarrierSetC2::compute_liveness_at_stubs() const {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   364
  ResourceMark rm;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   365
  Compile* const C = Compile::current();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   366
  Arena* const A = Thread::current()->resource_area();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   367
  PhaseCFG* const cfg = C->cfg();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   368
  PhaseRegAlloc* const regalloc = C->regalloc();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   369
  RegMask* const live = NEW_ARENA_ARRAY(A, RegMask, cfg->number_of_blocks() * sizeof(RegMask));
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   370
  ZBarrierSetAssembler* const bs = ZBarrierSet::assembler();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   371
  Block_List worklist;
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   372
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   373
  for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   374
    new ((void*)(live + i)) RegMask();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   375
    worklist.push(cfg->get_block(i));
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   376
  }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   377
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   378
  while (worklist.size() > 0) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   379
    const Block* const block = worklist.pop();
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   380
    RegMask& old_live = live[block->_pre_order];
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   381
    RegMask new_live;
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   382
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   383
    // Initialize to union of successors
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   384
    for (uint i = 0; i < block->_num_succs; i++) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   385
      const uint succ_id = block->_succs[i]->_pre_order;
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   386
      new_live.OR(live[succ_id]);
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   387
    }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   388
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   389
    // Walk block backwards, computing liveness
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   390
    for (int i = block->number_of_nodes() - 1; i >= 0; --i) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   391
      const Node* const node = block->get_node(i);
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   392
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   393
      // Remove def bits
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   394
      const OptoReg::Name first = bs->refine_register(node, regalloc->get_reg_first(node));
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   395
      const OptoReg::Name second = bs->refine_register(node, regalloc->get_reg_second(node));
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   396
      if (first != OptoReg::Bad) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   397
        new_live.Remove(first);
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   398
      }
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   399
      if (second != OptoReg::Bad) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   400
        new_live.Remove(second);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   401
      }
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   402
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   403
      // Add use bits
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   404
      for (uint j = 1; j < node->req(); ++j) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   405
        const Node* const use = node->in(j);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   406
        const OptoReg::Name first = bs->refine_register(use, regalloc->get_reg_first(use));
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   407
        const OptoReg::Name second = bs->refine_register(use, regalloc->get_reg_second(use));
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   408
        if (first != OptoReg::Bad) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   409
          new_live.Insert(first);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   410
        }
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   411
        if (second != OptoReg::Bad) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   412
          new_live.Insert(second);
55307
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   413
        }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   414
      }
ed12027517c0 8224675: Late GC barrier insertion for ZGC
neliasso
parents: 55123
diff changeset
   415
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   416
      // If this node tracks liveness, update it
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   417
      RegMask* const regs = barrier_set_state()->live(node);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   418
      if (regs != NULL) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   419
        regs->OR(new_live);
55321
ddda023e6f66 8225509: clean_catch_blocks must add preds first
neliasso
parents: 55307
diff changeset
   420
      }
ddda023e6f66 8225509: clean_catch_blocks must add preds first
neliasso
parents: 55307
diff changeset
   421
    }
ddda023e6f66 8225509: clean_catch_blocks must add preds first
neliasso
parents: 55307
diff changeset
   422
58516
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   423
    // Now at block top, see if we have any changes
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   424
    new_live.SUBTRACT(old_live);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   425
    if (new_live.is_NotEmpty()) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   426
      // Liveness has refined, update and propagate to prior blocks
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   427
      old_live.OR(new_live);
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   428
      for (uint i = 1; i < block->num_preds(); ++i) {
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   429
        Block* const pred = cfg->get_block_for_node(block->pred(i));
d376d86b0a01 8230565: ZGC: Redesign C2 load barrier to expand on the MachNode level
eosterlund
parents: 58273
diff changeset
   430
        worklist.push(pred);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   431
      }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   432
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   433
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   434
}
58931
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   435
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   436
const TypeFunc *oop_clone_Type() {
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   437
  // create input type (domain)
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   438
  const Type **fields = TypeTuple::fields(3);
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   439
  fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;  // src Object
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   440
  fields[TypeFunc::Parms+1] = TypeInstPtr::NOTNULL;  // dst Object
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   441
  fields[TypeFunc::Parms+2] = TypeInt::INT;          // Object size
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   442
  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+3, fields);
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   443
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   444
  // create result type (range)
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   445
  fields = TypeTuple::fields(0);
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   446
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   447
  const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   448
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   449
  return TypeFunc::make(domain, range);
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   450
}
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   451
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   452
void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   453
  Node *ctrl = ac->in(TypeFunc::Control);
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   454
  Node *mem = ac->in(TypeFunc::Memory);
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   455
  Node *src = ac->in(ArrayCopyNode::Src);
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   456
  Node *src_offset = ac->in(ArrayCopyNode::SrcPos);
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   457
  Node *dest = ac->in(ArrayCopyNode::Dest);
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   458
  Node *dest_offset = ac->in(ArrayCopyNode::DestPos);
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   459
  Node *length = ac->in(ArrayCopyNode::Length);
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   460
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   461
  assert (src_offset == NULL,  "for clone offsets should be null");
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   462
  assert (dest_offset == NULL, "for clone offsets should be null");
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   463
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   464
  if (src->bottom_type()->isa_instptr()) {
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   465
    // Instances must have all oop fiels healed before cloning - call runtime leaf
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   466
    const char *clonefunc_name = "clone_oop";
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   467
    address clonefunc_addr = ZBarrierSetRuntime::clone_oop_addr();
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   468
    const TypePtr *raw_adr_type = TypeRawPtr::BOTTOM;
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   469
    const TypeFunc *call_type = oop_clone_Type();
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   470
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   471
    Node *call = phase->make_leaf_call(ctrl, mem, call_type, clonefunc_addr, clonefunc_name, raw_adr_type, src, dest,
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   472
                                       length);
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   473
    phase->transform_later(call);
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   474
    phase->igvn().replace_node(ac, call);
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   475
  } else {
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   476
    assert(src->bottom_type()->isa_aryptr() != NULL, "Only arrays");
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   477
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   478
    // Clones of primitive arrays go here
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   479
    BarrierSetC2::clone_at_expansion(phase, ac);
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   480
  }
304c63b17b07 8232896: ZGC: Enable C2 clone intrinsic
neliasso
parents: 58516
diff changeset
   481
}