src/hotspot/share/gc/g1/g1SATBMarkQueueSet.cpp
author kbarrett
Wed, 26 Jun 2019 13:18:38 -0400
changeset 55498 e64383344f14
parent 54006 a421bdf22394
child 58059 baa4dd528de0
permissions -rw-r--r--
8225255: Make SATB qset lock-free Summary: Refactor PtrQueueSet, use lock-free stack for SATB completed buffers Reviewed-by: tschatzl, shade
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
51368
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
     1
/*
54006
a421bdf22394 8219613: Use NonJavaThread PtrQueues
kbarrett
parents: 52726
diff changeset
     2
 * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
51368
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
     3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
     4
 *
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
     5
 * This code is free software; you can redistribute it and/or modify it
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
     6
 * under the terms of the GNU General Public License version 2 only, as
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
     7
 * published by the Free Software Foundation.
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
     8
 *
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
     9
 * This code is distributed in the hope that it will be useful, but WITHOUT
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    12
 * version 2 for more details (a copy is included in the LICENSE file that
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    13
 * accompanied this code).
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    14
 *
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    15
 * You should have received a copy of the GNU General Public License version
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    16
 * 2 along with this work; if not, write to the Free Software Foundation,
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    18
 *
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    20
 * or visit www.oracle.com if you need additional information or have any
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    21
 * questions.
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    22
 *
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    23
 */
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    24
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    25
#include "precompiled.hpp"
51393
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    26
#include "gc/g1/g1CollectedHeap.inline.hpp"
51368
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    27
#include "gc/g1/g1SATBMarkQueueSet.hpp"
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    28
#include "gc/g1/g1ThreadLocalData.hpp"
51393
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    29
#include "gc/g1/heapRegion.hpp"
51441
2e91d927e00c 8154343: Make SATB related code available to other GCs
kbarrett
parents: 51401
diff changeset
    30
#include "gc/shared/satbMarkQueue.hpp"
51393
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    31
#include "oops/oop.hpp"
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    32
#include "utilities/debug.hpp"
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    33
#include "utilities/globalDefinitions.hpp"
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    34
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    35
G1SATBMarkQueueSet::G1SATBMarkQueueSet() : _g1h(NULL) {}
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    36
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    37
void G1SATBMarkQueueSet::initialize(G1CollectedHeap* g1h,
52582
6df094be7f58 8213352: Separate BufferNode allocation from PtrQueueSet
kbarrett
parents: 51441
diff changeset
    38
                                    BufferNode::Allocator* allocator,
52726
9cfa2e273b77 8214278: Cleanup process_completed_threshold and related state
kbarrett
parents: 52582
diff changeset
    39
                                    size_t process_completed_buffers_threshold,
54006
a421bdf22394 8219613: Use NonJavaThread PtrQueues
kbarrett
parents: 52726
diff changeset
    40
                                    uint buffer_enqueue_threshold_percentage) {
55498
e64383344f14 8225255: Make SATB qset lock-free
kbarrett
parents: 54006
diff changeset
    41
  SATBMarkQueueSet::initialize(allocator,
52726
9cfa2e273b77 8214278: Cleanup process_completed_threshold and related state
kbarrett
parents: 52582
diff changeset
    42
                               process_completed_buffers_threshold,
54006
a421bdf22394 8219613: Use NonJavaThread PtrQueues
kbarrett
parents: 52726
diff changeset
    43
                               buffer_enqueue_threshold_percentage);
51393
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    44
  _g1h = g1h;
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    45
}
51368
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    46
54006
a421bdf22394 8219613: Use NonJavaThread PtrQueues
kbarrett
parents: 52726
diff changeset
    47
void G1SATBMarkQueueSet::handle_zero_index_for_thread(Thread* t) {
51368
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    48
  G1ThreadLocalData::satb_mark_queue(t).handle_zero_index();
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    49
}
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    50
54006
a421bdf22394 8219613: Use NonJavaThread PtrQueues
kbarrett
parents: 52726
diff changeset
    51
SATBMarkQueue& G1SATBMarkQueueSet::satb_queue_for_thread(Thread* const t) const{
51368
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    52
  return G1ThreadLocalData::satb_mark_queue(t);
adcb0bb3d1e9 8209118: Abstract SATBMarkQueueSet's ThreadLocalData access
rkennke
parents:
diff changeset
    53
}
51393
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    54
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    55
// Return true if a SATB buffer entry refers to an object that
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    56
// requires marking.
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    57
//
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    58
// The entry must point into the G1 heap.  In particular, it must not
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    59
// be a NULL pointer.  NULL pointers are pre-filtered and never
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    60
// inserted into a SATB buffer.
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    61
//
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    62
// An entry that is below the NTAMS pointer for the containing heap
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    63
// region requires marking. Such an entry must point to a valid object.
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    64
//
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    65
// An entry that is at least the NTAMS pointer for the containing heap
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    66
// region might be any of the following, none of which should be marked.
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    67
//
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    68
// * A reference to an object allocated since marking started.
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    69
//   According to SATB, such objects are implicitly kept live and do
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    70
//   not need to be dealt with via SATB buffer processing.
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    71
//
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    72
// * A reference to a young generation object. Young objects are
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    73
//   handled separately and are not marked by concurrent marking.
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    74
//
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    75
// * A stale reference to a young generation object. If a young
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    76
//   generation object reference is recorded and not filtered out
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    77
//   before being moved by a young collection, the reference becomes
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    78
//   stale.
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    79
//
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    80
// * A stale reference to an eagerly reclaimed humongous object.  If a
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    81
//   humongous object is recorded and then reclaimed, the reference
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    82
//   becomes stale.
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    83
//
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    84
// The stale reference cases are implicitly handled by the NTAMS
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    85
// comparison. Because of the possibility of stale references, buffer
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    86
// processing must be somewhat circumspect and not assume entries
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    87
// in an unfiltered buffer refer to valid objects.
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    88
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    89
static inline bool requires_marking(const void* entry, G1CollectedHeap* g1h) {
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    90
  // Includes rejection of NULL pointers.
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    91
  assert(g1h->is_in_reserved(entry),
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    92
         "Non-heap pointer in SATB buffer: " PTR_FORMAT, p2i(entry));
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    93
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    94
  HeapRegion* region = g1h->heap_region_containing(entry);
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    95
  assert(region != NULL, "No region for " PTR_FORMAT, p2i(entry));
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    96
  if (entry >= region->next_top_at_mark_start()) {
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    97
    return false;
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    98
  }
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
    99
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   100
  assert(oopDesc::is_oop(oop(entry), true /* ignore mark word */),
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   101
         "Invalid oop in SATB buffer: " PTR_FORMAT, p2i(entry));
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   102
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   103
  return true;
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   104
}
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   105
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   106
static inline bool discard_entry(const void* entry, G1CollectedHeap* g1h) {
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   107
  return !requires_marking(entry, g1h) || g1h->is_marked_next((oop)entry);
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   108
}
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   109
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   110
// Workaround for not yet having std::bind.
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   111
class G1SATBMarkQueueFilterFn {
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   112
  G1CollectedHeap* _g1h;
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   113
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   114
public:
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   115
  G1SATBMarkQueueFilterFn(G1CollectedHeap* g1h) : _g1h(g1h) {}
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   116
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   117
  // Return true if entry should be filtered out (removed), false if
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   118
  // it should be retained.
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   119
  bool operator()(const void* entry) const {
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   120
    return discard_entry(entry, _g1h);
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   121
  }
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   122
};
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   123
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   124
void G1SATBMarkQueueSet::filter(SATBMarkQueue* queue) {
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   125
  assert(_g1h != NULL, "SATB queue set not initialized");
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   126
  apply_filter(G1SATBMarkQueueFilterFn(_g1h), queue);
cc8d309cd05a 8209345: Merge SATBMarkQueueFilter into SATBMarkQueueSet
kbarrett
parents: 51368
diff changeset
   127
}