src/hotspot/share/gc/shared/c2/barrierSetC2.cpp
author rkennke
Wed, 19 Sep 2018 21:31:33 +0200
changeset 51806 1ecc914fb707
parent 51485 0c7040d1d1ca
child 51826 e777e997e7c1
permissions -rw-r--r--
8210829: Modularize allocations in C2 Reviewed-by: kvn, roland
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
50180
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
     1
/*
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
     2
 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
     3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
     4
 *
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
     5
 * This code is free software; you can redistribute it and/or modify it
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
     6
 * under the terms of the GNU General Public License version 2 only, as
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
     7
 * published by the Free Software Foundation.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
     8
 *
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
     9
 * This code is distributed in the hope that it will be useful, but WITHOUT
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    12
 * version 2 for more details (a copy is included in the LICENSE file that
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    13
 * accompanied this code).
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    14
 *
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    15
 * You should have received a copy of the GNU General Public License version
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    16
 * 2 along with this work; if not, write to the Free Software Foundation,
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    18
 *
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    20
 * or visit www.oracle.com if you need additional information or have any
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    21
 * questions.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    22
 *
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    23
 */
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    24
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    25
#include "precompiled.hpp"
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    26
#include "gc/shared/c2/barrierSetC2.hpp"
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    27
#include "opto/arraycopynode.hpp"
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    28
#include "opto/graphKit.hpp"
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    29
#include "opto/idealKit.hpp"
51806
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
    30
#include "opto/macro.hpp"
50180
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    31
#include "opto/narrowptrnode.hpp"
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    32
#include "utilities/macros.hpp"
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    33
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    34
// By default this is a no-op.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    35
void BarrierSetC2::resolve_address(C2Access& access) const { }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    36
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    37
void* C2Access::barrier_set_state() const {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    38
  return _kit->barrier_set_state();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    39
}
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    40
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    41
bool C2Access::needs_cpu_membar() const {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    42
  bool mismatched = (_decorators & C2_MISMATCHED) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    43
  bool is_unordered = (_decorators & MO_UNORDERED) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    44
  bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0;
50599
ecc2af326b5f 8204939: Change Access nomenclature: root to native
kbarrett
parents: 50180
diff changeset
    45
  bool in_heap = (_decorators & IN_HEAP) != 0;
50180
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    46
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    47
  bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    48
  bool is_read = (_decorators & C2_READ_ACCESS) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    49
  bool is_atomic = is_read && is_write;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    50
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    51
  if (is_atomic) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    52
    // Atomics always need to be wrapped in CPU membars
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    53
    return true;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    54
  }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    55
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    56
  if (anonymous) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    57
    // We will need memory barriers unless we can determine a unique
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    58
    // alias category for this reference.  (Note:  If for some reason
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    59
    // the barriers get omitted and the unsafe reference begins to "pollute"
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    60
    // the alias analysis of the rest of the graph, either Compile::can_alias
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    61
    // or Compile::must_alias will throw a diagnostic assert.)
50599
ecc2af326b5f 8204939: Change Access nomenclature: root to native
kbarrett
parents: 50180
diff changeset
    62
    if (!in_heap || !is_unordered || (mismatched && !_addr.type()->isa_aryptr())) {
50180
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    63
      return true;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    64
    }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    65
  }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    66
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    67
  return false;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    68
}
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    69
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    70
Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    71
  DecoratorSet decorators = access.decorators();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    72
  GraphKit* kit = access.kit();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    73
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    74
  bool mismatched = (decorators & C2_MISMATCHED) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    75
  bool unaligned = (decorators & C2_UNALIGNED) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    76
  bool requires_atomic_access = (decorators & MO_UNORDERED) == 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    77
50599
ecc2af326b5f 8204939: Change Access nomenclature: root to native
kbarrett
parents: 50180
diff changeset
    78
  bool in_native = (decorators & IN_NATIVE) != 0;
ecc2af326b5f 8204939: Change Access nomenclature: root to native
kbarrett
parents: 50180
diff changeset
    79
  assert(!in_native, "not supported yet");
50180
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    80
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    81
  if (access.type() == T_DOUBLE) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    82
    Node* new_val = kit->dstore_rounding(val.node());
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    83
    val.set_node(new_val);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    84
  }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    85
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    86
  MemNode::MemOrd mo = access.mem_node_mo();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    87
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    88
  Node* store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), access.type(),
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    89
                                     access.addr().type(), mo, requires_atomic_access, unaligned, mismatched);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    90
  access.set_raw_access(store);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    91
  return store;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    92
}
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    93
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    94
Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    95
  DecoratorSet decorators = access.decorators();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    96
  GraphKit* kit = access.kit();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    97
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    98
  Node* adr = access.addr().node();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
    99
  const TypePtr* adr_type = access.addr().type();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   100
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   101
  bool mismatched = (decorators & C2_MISMATCHED) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   102
  bool requires_atomic_access = (decorators & MO_UNORDERED) == 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   103
  bool unaligned = (decorators & C2_UNALIGNED) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   104
  bool control_dependent = (decorators & C2_CONTROL_DEPENDENT_LOAD) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   105
  bool pinned = (decorators & C2_PINNED_LOAD) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   106
50599
ecc2af326b5f 8204939: Change Access nomenclature: root to native
kbarrett
parents: 50180
diff changeset
   107
  bool in_native = (decorators & IN_NATIVE) != 0;
50180
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   108
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   109
  MemNode::MemOrd mo = access.mem_node_mo();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   110
  LoadNode::ControlDependency dep = pinned ? LoadNode::Pinned : LoadNode::DependsOnlyOnTest;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   111
  Node* control = control_dependent ? kit->control() : NULL;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   112
51485
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 51482
diff changeset
   113
  Node* load;
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 51482
diff changeset
   114
  if (in_native) {
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 51482
diff changeset
   115
    load = kit->make_load(control, adr, val_type, access.type(), mo);
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 51482
diff changeset
   116
  } else {
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 51482
diff changeset
   117
    load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo,
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 51482
diff changeset
   118
                          dep, requires_atomic_access, unaligned, mismatched);
0c7040d1d1ca 8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents: 51482
diff changeset
   119
  }
50180
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   120
  access.set_raw_access(load);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   121
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   122
  return load;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   123
}
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   124
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   125
class C2AccessFence: public StackObj {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   126
  C2Access& _access;
51482
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   127
  Node* _leading_membar;
50180
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   128
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   129
public:
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   130
  C2AccessFence(C2Access& access) :
51482
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   131
    _access(access), _leading_membar(NULL) {
50180
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   132
    GraphKit* kit = access.kit();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   133
    DecoratorSet decorators = access.decorators();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   134
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   135
    bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   136
    bool is_read = (decorators & C2_READ_ACCESS) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   137
    bool is_atomic = is_read && is_write;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   138
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   139
    bool is_volatile = (decorators & MO_SEQ_CST) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   140
    bool is_release = (decorators & MO_RELEASE) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   141
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   142
    if (is_atomic) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   143
      // Memory-model-wise, a LoadStore acts like a little synchronized
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   144
      // block, so needs barriers on each side.  These don't translate
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   145
      // into actual barriers on most machines, but we still need rest of
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   146
      // compiler to respect ordering.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   147
      if (is_release) {
51482
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   148
        _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
50180
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   149
      } else if (is_volatile) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   150
        if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
51482
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   151
          _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
50180
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   152
        } else {
51482
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   153
          _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
50180
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   154
        }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   155
      }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   156
    } else if (is_write) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   157
      // If reference is volatile, prevent following memory ops from
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   158
      // floating down past the volatile write.  Also prevents commoning
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   159
      // another volatile read.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   160
      if (is_volatile || is_release) {
51482
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   161
        _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
50180
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   162
      }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   163
    } else {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   164
      // Memory barrier to prevent normal and 'unsafe' accesses from
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   165
      // bypassing each other.  Happens after null checks, so the
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   166
      // exception paths do not take memory state from the memory barrier,
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   167
      // so there's no problems making a strong assert about mixing users
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   168
      // of safe & unsafe memory.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   169
      if (is_volatile && support_IRIW_for_not_multiple_copy_atomic_cpu) {
51482
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   170
        _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
50180
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   171
      }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   172
    }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   173
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   174
    if (access.needs_cpu_membar()) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   175
      kit->insert_mem_bar(Op_MemBarCPUOrder);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   176
    }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   177
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   178
    if (is_atomic) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   179
      // 4984716: MemBars must be inserted before this
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   180
      //          memory node in order to avoid a false
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   181
      //          dependency which will confuse the scheduler.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   182
      access.set_memory();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   183
    }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   184
  }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   185
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   186
  ~C2AccessFence() {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   187
    GraphKit* kit = _access.kit();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   188
    DecoratorSet decorators = _access.decorators();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   189
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   190
    bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   191
    bool is_read = (decorators & C2_READ_ACCESS) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   192
    bool is_atomic = is_read && is_write;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   193
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   194
    bool is_volatile = (decorators & MO_SEQ_CST) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   195
    bool is_acquire = (decorators & MO_ACQUIRE) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   196
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   197
    // If reference is volatile, prevent following volatiles ops from
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   198
    // floating up before the volatile access.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   199
    if (_access.needs_cpu_membar()) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   200
      kit->insert_mem_bar(Op_MemBarCPUOrder);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   201
    }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   202
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   203
    if (is_atomic) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   204
      if (is_acquire || is_volatile) {
51482
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   205
        Node* n = _access.raw_access();
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   206
        Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   207
        if (_leading_membar != NULL) {
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   208
          MemBarNode::set_load_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   209
        }
50180
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   210
      }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   211
    } else if (is_write) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   212
      // If not multiple copy atomic, we do the MemBarVolatile before the load.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   213
      if (is_volatile && !support_IRIW_for_not_multiple_copy_atomic_cpu) {
51482
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   214
        Node* n = _access.raw_access();
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   215
        Node* mb = kit->insert_mem_bar(Op_MemBarVolatile, n); // Use fat membar
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   216
        if (_leading_membar != NULL) {
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   217
          MemBarNode::set_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   218
        }
50180
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   219
      }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   220
    } else {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   221
      if (is_volatile || is_acquire) {
51482
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   222
        Node* n = _access.raw_access();
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   223
        assert(_leading_membar == NULL || support_IRIW_for_not_multiple_copy_atomic_cpu, "no leading membar expected");
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   224
        Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
d7029542d67a 8209420: Track membars for volatile accesses so they can be properly optimized
roland
parents: 50599
diff changeset
   225
        mb->as_MemBar()->set_trailing_load();
50180
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   226
      }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   227
    }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   228
  }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   229
};
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   230
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   231
Node* BarrierSetC2::store_at(C2Access& access, C2AccessValue& val) const {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   232
  C2AccessFence fence(access);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   233
  resolve_address(access);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   234
  return store_at_resolved(access, val);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   235
}
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   236
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   237
Node* BarrierSetC2::load_at(C2Access& access, const Type* val_type) const {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   238
  C2AccessFence fence(access);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   239
  resolve_address(access);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   240
  return load_at_resolved(access, val_type);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   241
}
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   242
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   243
MemNode::MemOrd C2Access::mem_node_mo() const {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   244
  bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   245
  bool is_read = (_decorators & C2_READ_ACCESS) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   246
  if ((_decorators & MO_SEQ_CST) != 0) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   247
    if (is_write && is_read) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   248
      // For atomic operations
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   249
      return MemNode::seqcst;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   250
    } else if (is_write) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   251
      return MemNode::release;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   252
    } else {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   253
      assert(is_read, "what else?");
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   254
      return MemNode::acquire;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   255
    }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   256
  } else if ((_decorators & MO_RELEASE) != 0) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   257
    return MemNode::release;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   258
  } else if ((_decorators & MO_ACQUIRE) != 0) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   259
    return MemNode::acquire;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   260
  } else if (is_write) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   261
    // Volatile fields need releasing stores.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   262
    // Non-volatile fields also need releasing stores if they hold an
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   263
    // object reference, because the object reference might point to
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   264
    // a freshly created object.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   265
    // Conservatively release stores of object references.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   266
    return StoreNode::release_if_reference(_type);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   267
  } else {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   268
    return MemNode::unordered;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   269
  }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   270
}
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   271
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   272
void C2Access::fixup_decorators() {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   273
  bool default_mo = (_decorators & MO_DECORATOR_MASK) == 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   274
  bool is_unordered = (_decorators & MO_UNORDERED) != 0 || default_mo;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   275
  bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   276
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   277
  bool is_read = (_decorators & C2_READ_ACCESS) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   278
  bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   279
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   280
  if (AlwaysAtomicAccesses && is_unordered) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   281
    _decorators &= ~MO_DECORATOR_MASK; // clear the MO bits
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   282
    _decorators |= MO_RELAXED; // Force the MO_RELAXED decorator with AlwaysAtomicAccess
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   283
  }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   284
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   285
  _decorators = AccessInternal::decorator_fixup(_decorators);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   286
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   287
  if (is_read && !is_write && anonymous) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   288
    // To be valid, unsafe loads may depend on other conditions than
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   289
    // the one that guards them: pin the Load node
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   290
    _decorators |= C2_CONTROL_DEPENDENT_LOAD;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   291
    _decorators |= C2_PINNED_LOAD;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   292
    const TypePtr* adr_type = _addr.type();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   293
    Node* adr = _addr.node();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   294
    if (!needs_cpu_membar() && adr_type->isa_instptr()) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   295
      assert(adr_type->meet(TypePtr::NULL_PTR) != adr_type->remove_speculative(), "should be not null");
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   296
      intptr_t offset = Type::OffsetBot;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   297
      AddPNode::Ideal_base_and_offset(adr, &_kit->gvn(), offset);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   298
      if (offset >= 0) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   299
        int s = Klass::layout_helper_size_in_bytes(adr_type->isa_instptr()->klass()->layout_helper());
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   300
        if (offset < s) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   301
          // Guaranteed to be a valid access, no need to pin it
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   302
          _decorators ^= C2_CONTROL_DEPENDENT_LOAD;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   303
          _decorators ^= C2_PINNED_LOAD;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   304
        }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   305
      }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   306
    }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   307
  }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   308
}
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   309
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   310
//--------------------------- atomic operations---------------------------------
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   311
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   312
static void pin_atomic_op(C2AtomicAccess& access) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   313
  if (!access.needs_pinning()) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   314
    return;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   315
  }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   316
  // SCMemProjNodes represent the memory state of a LoadStore. Their
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   317
  // main role is to prevent LoadStore nodes from being optimized away
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   318
  // when their results aren't used.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   319
  GraphKit* kit = access.kit();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   320
  Node* load_store = access.raw_access();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   321
  assert(load_store != NULL, "must pin atomic op");
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   322
  Node* proj = kit->gvn().transform(new SCMemProjNode(load_store));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   323
  kit->set_memory(proj, access.alias_idx());
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   324
}
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   325
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   326
void C2AtomicAccess::set_memory() {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   327
  Node *mem = _kit->memory(_alias_idx);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   328
  _memory = mem;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   329
}
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   330
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   331
Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val,
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   332
                                                   Node* new_val, const Type* value_type) const {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   333
  GraphKit* kit = access.kit();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   334
  MemNode::MemOrd mo = access.mem_node_mo();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   335
  Node* mem = access.memory();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   336
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   337
  Node* adr = access.addr().node();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   338
  const TypePtr* adr_type = access.addr().type();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   339
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   340
  Node* load_store = NULL;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   341
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   342
  if (access.is_oop()) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   343
#ifdef _LP64
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   344
    if (adr->bottom_type()->is_ptr_to_narrowoop()) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   345
      Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   346
      Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   347
      load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   348
    } else
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   349
#endif
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   350
    {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   351
      load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   352
    }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   353
  } else {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   354
    switch (access.type()) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   355
      case T_BYTE: {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   356
        load_store = kit->gvn().transform(new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   357
        break;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   358
      }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   359
      case T_SHORT: {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   360
        load_store = kit->gvn().transform(new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   361
        break;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   362
      }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   363
      case T_INT: {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   364
        load_store = kit->gvn().transform(new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   365
        break;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   366
      }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   367
      case T_LONG: {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   368
        load_store = kit->gvn().transform(new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   369
        break;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   370
      }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   371
      default:
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   372
        ShouldNotReachHere();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   373
    }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   374
  }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   375
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   376
  access.set_raw_access(load_store);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   377
  pin_atomic_op(access);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   378
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   379
#ifdef _LP64
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   380
  if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   381
    return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   382
  }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   383
#endif
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   384
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   385
  return load_store;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   386
}
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   387
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   388
Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val,
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   389
                                                    Node* new_val, const Type* value_type) const {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   390
  GraphKit* kit = access.kit();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   391
  DecoratorSet decorators = access.decorators();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   392
  MemNode::MemOrd mo = access.mem_node_mo();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   393
  Node* mem = access.memory();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   394
  bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   395
  Node* load_store = NULL;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   396
  Node* adr = access.addr().node();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   397
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   398
  if (access.is_oop()) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   399
#ifdef _LP64
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   400
    if (adr->bottom_type()->is_ptr_to_narrowoop()) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   401
      Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   402
      Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   403
      if (is_weak_cas) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   404
        load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   405
      } else {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   406
        load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   407
      }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   408
    } else
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   409
#endif
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   410
    {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   411
      if (is_weak_cas) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   412
        load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   413
      } else {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   414
        load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   415
      }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   416
    }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   417
  } else {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   418
    switch(access.type()) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   419
      case T_BYTE: {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   420
        if (is_weak_cas) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   421
          load_store = kit->gvn().transform(new WeakCompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   422
        } else {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   423
          load_store = kit->gvn().transform(new CompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   424
        }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   425
        break;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   426
      }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   427
      case T_SHORT: {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   428
        if (is_weak_cas) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   429
          load_store = kit->gvn().transform(new WeakCompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   430
        } else {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   431
          load_store = kit->gvn().transform(new CompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   432
        }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   433
        break;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   434
      }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   435
      case T_INT: {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   436
        if (is_weak_cas) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   437
          load_store = kit->gvn().transform(new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   438
        } else {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   439
          load_store = kit->gvn().transform(new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   440
        }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   441
        break;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   442
      }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   443
      case T_LONG: {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   444
        if (is_weak_cas) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   445
          load_store = kit->gvn().transform(new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   446
        } else {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   447
          load_store = kit->gvn().transform(new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   448
        }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   449
        break;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   450
      }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   451
      default:
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   452
        ShouldNotReachHere();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   453
    }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   454
  }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   455
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   456
  access.set_raw_access(load_store);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   457
  pin_atomic_op(access);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   458
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   459
  return load_store;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   460
}
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   461
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   462
Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* value_type) const {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   463
  GraphKit* kit = access.kit();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   464
  Node* mem = access.memory();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   465
  Node* adr = access.addr().node();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   466
  const TypePtr* adr_type = access.addr().type();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   467
  Node* load_store = NULL;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   468
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   469
  if (access.is_oop()) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   470
#ifdef _LP64
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   471
    if (adr->bottom_type()->is_ptr_to_narrowoop()) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   472
      Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   473
      load_store = kit->gvn().transform(new GetAndSetNNode(kit->control(), mem, adr, newval_enc, adr_type, value_type->make_narrowoop()));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   474
    } else
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   475
#endif
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   476
    {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   477
      load_store = kit->gvn().transform(new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr()));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   478
    }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   479
  } else  {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   480
    switch (access.type()) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   481
      case T_BYTE:
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   482
        load_store = kit->gvn().transform(new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   483
        break;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   484
      case T_SHORT:
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   485
        load_store = kit->gvn().transform(new GetAndSetSNode(kit->control(), mem, adr, new_val, adr_type));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   486
        break;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   487
      case T_INT:
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   488
        load_store = kit->gvn().transform(new GetAndSetINode(kit->control(), mem, adr, new_val, adr_type));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   489
        break;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   490
      case T_LONG:
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   491
        load_store = kit->gvn().transform(new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   492
        break;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   493
      default:
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   494
        ShouldNotReachHere();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   495
    }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   496
  }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   497
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   498
  access.set_raw_access(load_store);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   499
  pin_atomic_op(access);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   500
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   501
#ifdef _LP64
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   502
  if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   503
    return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   504
  }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   505
#endif
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   506
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   507
  return load_store;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   508
}
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   509
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   510
Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* value_type) const {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   511
  Node* load_store = NULL;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   512
  GraphKit* kit = access.kit();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   513
  Node* adr = access.addr().node();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   514
  const TypePtr* adr_type = access.addr().type();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   515
  Node* mem = access.memory();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   516
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   517
  switch(access.type()) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   518
    case T_BYTE:
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   519
      load_store = kit->gvn().transform(new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   520
      break;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   521
    case T_SHORT:
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   522
      load_store = kit->gvn().transform(new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   523
      break;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   524
    case T_INT:
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   525
      load_store = kit->gvn().transform(new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   526
      break;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   527
    case T_LONG:
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   528
      load_store = kit->gvn().transform(new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   529
      break;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   530
    default:
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   531
      ShouldNotReachHere();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   532
  }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   533
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   534
  access.set_raw_access(load_store);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   535
  pin_atomic_op(access);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   536
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   537
  return load_store;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   538
}
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   539
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   540
Node* BarrierSetC2::atomic_cmpxchg_val_at(C2AtomicAccess& access, Node* expected_val,
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   541
                                          Node* new_val, const Type* value_type) const {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   542
  C2AccessFence fence(access);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   543
  resolve_address(access);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   544
  return atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   545
}
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   546
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   547
Node* BarrierSetC2::atomic_cmpxchg_bool_at(C2AtomicAccess& access, Node* expected_val,
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   548
                                           Node* new_val, const Type* value_type) const {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   549
  C2AccessFence fence(access);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   550
  resolve_address(access);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   551
  return atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   552
}
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   553
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   554
Node* BarrierSetC2::atomic_xchg_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   555
  C2AccessFence fence(access);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   556
  resolve_address(access);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   557
  return atomic_xchg_at_resolved(access, new_val, value_type);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   558
}
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   559
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   560
Node* BarrierSetC2::atomic_add_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   561
  C2AccessFence fence(access);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   562
  resolve_address(access);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   563
  return atomic_add_at_resolved(access, new_val, value_type);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   564
}
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   565
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   566
void BarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   567
  // Exclude the header but include array length to copy by 8 bytes words.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   568
  // Can't use base_offset_in_bytes(bt) since basic type is unknown.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   569
  int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   570
                            instanceOopDesc::base_offset_in_bytes();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   571
  // base_off:
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   572
  // 8  - 32-bit VM
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   573
  // 12 - 64-bit VM, compressed klass
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   574
  // 16 - 64-bit VM, normal klass
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   575
  if (base_off % BytesPerLong != 0) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   576
    assert(UseCompressedClassPointers, "");
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   577
    if (is_array) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   578
      // Exclude length to copy by 8 bytes words.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   579
      base_off += sizeof(int);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   580
    } else {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   581
      // Include klass to copy by 8 bytes words.
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   582
      base_off = instanceOopDesc::klass_offset_in_bytes();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   583
    }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   584
    assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   585
  }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   586
  Node* src_base  = kit->basic_plus_adr(src,  base_off);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   587
  Node* dst_base = kit->basic_plus_adr(dst, base_off);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   588
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   589
  // Compute the length also, if needed:
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   590
  Node* countx = size;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   591
  countx = kit->gvn().transform(new SubXNode(countx, kit->MakeConX(base_off)));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   592
  countx = kit->gvn().transform(new URShiftXNode(countx, kit->intcon(LogBytesPerLong) ));
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   593
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   594
  const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   595
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   596
  ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, NULL, dst_base, NULL, countx, false, false);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   597
  ac->set_clonebasic();
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   598
  Node* n = kit->gvn().transform(ac);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   599
  if (n == ac) {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   600
    kit->set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   601
  } else {
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   602
    kit->set_all_memory(n);
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   603
  }
ffa644980dff 8202377: Modularize C2 GC barriers
eosterlund
parents:
diff changeset
   604
}
51806
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   605
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   606
Node* BarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes,
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   607
                                 Node*& i_o, Node*& needgc_ctrl,
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   608
                                 Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   609
                                 intx prefetch_lines) const {
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   610
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   611
  Node* eden_top_adr;
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   612
  Node* eden_end_adr;
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   613
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   614
  macro->set_eden_pointers(eden_top_adr, eden_end_adr);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   615
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   616
  // Load Eden::end.  Loop invariant and hoisted.
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   617
  //
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   618
  // Note: We set the control input on "eden_end" and "old_eden_top" when using
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   619
  //       a TLAB to work around a bug where these values were being moved across
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   620
  //       a safepoint.  These are not oops, so they cannot be include in the oop
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   621
  //       map, but they can be changed by a GC.   The proper way to fix this would
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   622
  //       be to set the raw memory state when generating a  SafepointNode.  However
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   623
  //       this will require extensive changes to the loop optimization in order to
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   624
  //       prevent a degradation of the optimization.
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   625
  //       See comment in memnode.hpp, around line 227 in class LoadPNode.
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   626
  Node *eden_end = macro->make_load(ctrl, mem, eden_end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   627
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   628
  // We need a Region for the loop-back contended case.
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   629
  enum { fall_in_path = 1, contended_loopback_path = 2 };
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   630
  Node *contended_region;
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   631
  Node *contended_phi_rawmem;
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   632
  if (UseTLAB) {
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   633
    contended_region = toobig_false;
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   634
    contended_phi_rawmem = mem;
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   635
  } else {
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   636
    contended_region = new RegionNode(3);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   637
    contended_phi_rawmem = new PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   638
    // Now handle the passing-too-big test.  We fall into the contended
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   639
    // loop-back merge point.
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   640
    contended_region    ->init_req(fall_in_path, toobig_false);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   641
    contended_phi_rawmem->init_req(fall_in_path, mem);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   642
    macro->transform_later(contended_region);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   643
    macro->transform_later(contended_phi_rawmem);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   644
  }
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   645
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   646
  // Load(-locked) the heap top.
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   647
  // See note above concerning the control input when using a TLAB
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   648
  Node *old_eden_top = UseTLAB
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   649
    ? new LoadPNode      (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered)
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   650
    : new LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr, MemNode::acquire);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   651
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   652
  macro->transform_later(old_eden_top);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   653
  // Add to heap top to get a new heap top
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   654
  Node *new_eden_top = new AddPNode(macro->top(), old_eden_top, size_in_bytes);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   655
  macro->transform_later(new_eden_top);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   656
  // Check for needing a GC; compare against heap end
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   657
  Node *needgc_cmp = new CmpPNode(new_eden_top, eden_end);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   658
  macro->transform_later(needgc_cmp);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   659
  Node *needgc_bol = new BoolNode(needgc_cmp, BoolTest::ge);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   660
  macro->transform_later(needgc_bol);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   661
  IfNode *needgc_iff = new IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   662
  macro->transform_later(needgc_iff);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   663
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   664
  // Plug the failing-heap-space-need-gc test into the slow-path region
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   665
  Node *needgc_true = new IfTrueNode(needgc_iff);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   666
  macro->transform_later(needgc_true);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   667
  needgc_ctrl = needgc_true;
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   668
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   669
  // No need for a GC.  Setup for the Store-Conditional
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   670
  Node *needgc_false = new IfFalseNode(needgc_iff);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   671
  macro->transform_later(needgc_false);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   672
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   673
  i_o = macro->prefetch_allocation(i_o, needgc_false, contended_phi_rawmem,
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   674
                                   old_eden_top, new_eden_top, prefetch_lines);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   675
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   676
  Node* fast_oop = old_eden_top;
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   677
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   678
  // Store (-conditional) the modified eden top back down.
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   679
  // StorePConditional produces flags for a test PLUS a modified raw
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   680
  // memory state.
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   681
  if (UseTLAB) {
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   682
    Node* store_eden_top =
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   683
      new StorePNode(needgc_false, contended_phi_rawmem, eden_top_adr,
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   684
                     TypeRawPtr::BOTTOM, new_eden_top, MemNode::unordered);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   685
    macro->transform_later(store_eden_top);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   686
    fast_oop_ctrl = needgc_false; // No contention, so this is the fast path
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   687
    fast_oop_rawmem = store_eden_top;
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   688
  } else {
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   689
    Node* store_eden_top =
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   690
      new StorePConditionalNode(needgc_false, contended_phi_rawmem, eden_top_adr,
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   691
                                new_eden_top, fast_oop/*old_eden_top*/);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   692
    macro->transform_later(store_eden_top);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   693
    Node *contention_check = new BoolNode(store_eden_top, BoolTest::ne);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   694
    macro->transform_later(contention_check);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   695
    store_eden_top = new SCMemProjNode(store_eden_top);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   696
    macro->transform_later(store_eden_top);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   697
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   698
    // If not using TLABs, check to see if there was contention.
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   699
    IfNode *contention_iff = new IfNode (needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   700
    macro->transform_later(contention_iff);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   701
    Node *contention_true = new IfTrueNode(contention_iff);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   702
    macro->transform_later(contention_true);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   703
    // If contention, loopback and try again.
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   704
    contended_region->init_req(contended_loopback_path, contention_true);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   705
    contended_phi_rawmem->init_req(contended_loopback_path, store_eden_top);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   706
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   707
    // Fast-path succeeded with no contention!
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   708
    Node *contention_false = new IfFalseNode(contention_iff);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   709
    macro->transform_later(contention_false);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   710
    fast_oop_ctrl = contention_false;
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   711
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   712
    // Bump total allocated bytes for this thread
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   713
    Node* thread = new ThreadLocalNode();
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   714
    macro->transform_later(thread);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   715
    Node* alloc_bytes_adr = macro->basic_plus_adr(macro->top()/*not oop*/, thread,
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   716
                                                  in_bytes(JavaThread::allocated_bytes_offset()));
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   717
    Node* alloc_bytes = macro->make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   718
                                         0, TypeLong::LONG, T_LONG);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   719
#ifdef _LP64
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   720
    Node* alloc_size = size_in_bytes;
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   721
#else
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   722
    Node* alloc_size = new ConvI2LNode(size_in_bytes);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   723
    macro->transform_later(alloc_size);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   724
#endif
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   725
    Node* new_alloc_bytes = new AddLNode(alloc_bytes, alloc_size);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   726
    macro->transform_later(new_alloc_bytes);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   727
    fast_oop_rawmem = macro->make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   728
                                        0, new_alloc_bytes, T_LONG);
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   729
  }
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   730
  return fast_oop;
1ecc914fb707 8210829: Modularize allocations in C2
rkennke
parents: 51485
diff changeset
   731
}