author | stefank |
Tue, 26 Nov 2019 10:47:46 +0100 | |
changeset 59290 | 97d13893ec3c |
parent 59233 | bd9dba789919 |
permissions | -rw-r--r-- |
1374 | 1 |
/* |
53482 | 2 |
* Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved. |
1374 | 3 |
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
4 |
* |
|
5 |
* This code is free software; you can redistribute it and/or modify it |
|
6 |
* under the terms of the GNU General Public License version 2 only, as |
|
7 |
* published by the Free Software Foundation. |
|
8 |
* |
|
9 |
* This code is distributed in the hope that it will be useful, but WITHOUT |
|
10 |
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|
11 |
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
|
12 |
* version 2 for more details (a copy is included in the LICENSE file that |
|
13 |
* accompanied this code). |
|
14 |
* |
|
15 |
* You should have received a copy of the GNU General Public License version |
|
16 |
* 2 along with this work; if not, write to the Free Software Foundation, |
|
17 |
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
|
18 |
* |
|
5547
f4b087cbb361
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
4481
diff
changeset
|
19 |
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
f4b087cbb361
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
4481
diff
changeset
|
20 |
* or visit www.oracle.com if you need additional information or have any |
f4b087cbb361
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
4481
diff
changeset
|
21 |
* questions. |
1374 | 22 |
* |
23 |
*/ |
|
24 |
||
7397 | 25 |
#include "precompiled.hpp" |
57890
6bb824c45df1
8230192: Rename G1RedirtyCardsBufferList to G1BufferNodeList
kbarrett
parents:
57507
diff
changeset
|
26 |
#include "gc/g1/g1BufferNodeList.hpp" |
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
27 |
#include "gc/g1/g1CardTableEntryClosure.hpp" |
30764 | 28 |
#include "gc/g1/g1CollectedHeap.inline.hpp" |
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
29 |
#include "gc/g1/g1DirtyCardQueue.hpp" |
53482 | 30 |
#include "gc/g1/g1FreeIdSet.hpp" |
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
31 |
#include "gc/g1/g1RedirtyCardsQueue.hpp" |
46653 | 32 |
#include "gc/g1/g1RemSet.hpp" |
49752
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49333
diff
changeset
|
33 |
#include "gc/g1/g1ThreadLocalData.hpp" |
30764 | 34 |
#include "gc/g1/heapRegionRemSet.hpp" |
52576
367ca5f32505
8213829: Remove circular dependency between g1CollectedHeap and g1ConcurrentMark
manc
parents:
49752
diff
changeset
|
35 |
#include "gc/shared/suspendibleThreadSet.hpp" |
30764 | 36 |
#include "gc/shared/workgroup.hpp" |
52726
9cfa2e273b77
8214278: Cleanup process_completed_threshold and related state
kbarrett
parents:
52684
diff
changeset
|
37 |
#include "runtime/flags/flagSetting.hpp" |
7397 | 38 |
#include "runtime/mutexLocker.hpp" |
59290
97d13893ec3c
8234748: Clean up atomic and orderAccess includes
stefank
parents:
59233
diff
changeset
|
39 |
#include "runtime/orderAccess.hpp" |
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
40 |
#include "runtime/os.hpp" |
7397 | 41 |
#include "runtime/safepoint.hpp" |
14583
d70ee55535f4
8003935: Simplify the needed includes for using Thread::current()
stefank
parents:
10565
diff
changeset
|
42 |
#include "runtime/thread.inline.hpp" |
48105
8d15b1369c7a
8167108: inconsistent handling of SR_lock can lead to crashes
dcubed
parents:
47634
diff
changeset
|
43 |
#include "runtime/threadSMR.hpp" |
59233 | 44 |
#include "utilities/quickSort.hpp" |
1374 | 45 |
|
54255 | 46 |
G1DirtyCardQueue::G1DirtyCardQueue(G1DirtyCardQueueSet* qset) : |
33761
329db4b51480
6899049: G1: Clean up code in ptrQueue.[ch]pp and ptrQueue.inline.hpp
kbarrett
parents:
30764
diff
changeset
|
47 |
// Dirty card queues are always active, so we create them with their |
329db4b51480
6899049: G1: Clean up code in ptrQueue.[ch]pp and ptrQueue.inline.hpp
kbarrett
parents:
30764
diff
changeset
|
48 |
// active field set to true. |
54255 | 49 |
PtrQueue(qset, true /* active */) |
33761
329db4b51480
6899049: G1: Clean up code in ptrQueue.[ch]pp and ptrQueue.inline.hpp
kbarrett
parents:
30764
diff
changeset
|
50 |
{ } |
329db4b51480
6899049: G1: Clean up code in ptrQueue.[ch]pp and ptrQueue.inline.hpp
kbarrett
parents:
30764
diff
changeset
|
51 |
|
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
52 |
G1DirtyCardQueue::~G1DirtyCardQueue() { |
54255 | 53 |
flush(); |
33761
329db4b51480
6899049: G1: Clean up code in ptrQueue.[ch]pp and ptrQueue.inline.hpp
kbarrett
parents:
30764
diff
changeset
|
54 |
} |
329db4b51480
6899049: G1: Clean up code in ptrQueue.[ch]pp and ptrQueue.inline.hpp
kbarrett
parents:
30764
diff
changeset
|
55 |
|
54970
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
56 |
void G1DirtyCardQueue::handle_completed_buffer() { |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
57 |
assert(_buf != NULL, "precondition"); |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
58 |
BufferNode* node = BufferNode::make_node_from_buffer(_buf, index()); |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
59 |
G1DirtyCardQueueSet* dcqs = dirty_card_qset(); |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
60 |
if (dcqs->process_or_enqueue_completed_buffer(node)) { |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
61 |
reset(); // Buffer fully processed, reset index. |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
62 |
} else { |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
63 |
allocate_buffer(); // Buffer enqueued, get a new one. |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
64 |
} |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
65 |
} |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
66 |
|
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
67 |
// Assumed to be zero by concurrent threads. |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
68 |
static uint par_ids_start() { return 0; } |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
69 |
|
58059
baa4dd528de0
8221361: Eliminate two-phase initialization for PtrQueueSet classes
kbarrett
parents:
58033
diff
changeset
|
70 |
G1DirtyCardQueueSet::G1DirtyCardQueueSet(Monitor* cbl_mon, |
baa4dd528de0
8221361: Eliminate two-phase initialization for PtrQueueSet classes
kbarrett
parents:
58033
diff
changeset
|
71 |
BufferNode::Allocator* allocator) : |
baa4dd528de0
8221361: Eliminate two-phase initialization for PtrQueueSet classes
kbarrett
parents:
58033
diff
changeset
|
72 |
PtrQueueSet(allocator), |
baa4dd528de0
8221361: Eliminate two-phase initialization for PtrQueueSet classes
kbarrett
parents:
58033
diff
changeset
|
73 |
_cbl_mon(cbl_mon), |
55498 | 74 |
_completed_buffers_head(NULL), |
75 |
_completed_buffers_tail(NULL), |
|
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
76 |
_num_cards(0), |
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
77 |
_process_cards_threshold(ProcessCardsThresholdNever), |
55498 | 78 |
_process_completed_buffers(false), |
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
79 |
_max_cards(MaxCardsUnlimited), |
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
80 |
_max_cards_padding(0), |
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
81 |
_free_ids(par_ids_start(), num_par_ids()), |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
82 |
_mutator_refined_cards_counters(NEW_C_HEAP_ARRAY(size_t, num_par_ids(), mtGC)) |
1374 | 83 |
{ |
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
84 |
::memset(_mutator_refined_cards_counters, 0, num_par_ids() * sizeof(size_t)); |
1374 | 85 |
_all_active = true; |
86 |
} |
|
87 |
||
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
88 |
G1DirtyCardQueueSet::~G1DirtyCardQueueSet() { |
55498 | 89 |
abandon_completed_buffers(); |
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
90 |
FREE_C_HEAP_ARRAY(size_t, _mutator_refined_cards_counters); |
53482 | 91 |
} |
92 |
||
2882
d508a8bac491
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
2881
diff
changeset
|
93 |
// Determines how many mutator threads can process the buffers in parallel. |
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
94 |
uint G1DirtyCardQueueSet::num_par_ids() { |
40012
f69cfe79fe98
8161993: G1 crashes if active_processor_count changes during startup
tschatzl
parents:
37197
diff
changeset
|
95 |
return (uint)os::initial_active_processor_count(); |
1374 | 96 |
} |
97 |
||
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
98 |
size_t G1DirtyCardQueueSet::total_mutator_refined_cards() const { |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
99 |
size_t sum = 0; |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
100 |
for (uint i = 0; i < num_par_ids(); ++i) { |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
101 |
sum += _mutator_refined_cards_counters[i]; |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
102 |
} |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
103 |
return sum; |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
104 |
} |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
105 |
|
54006 | 106 |
void G1DirtyCardQueueSet::handle_zero_index_for_thread(Thread* t) { |
49752
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49333
diff
changeset
|
107 |
G1ThreadLocalData::dirty_card_queue(t).handle_zero_index(); |
1374 | 108 |
} |
109 |
||
55498 | 110 |
void G1DirtyCardQueueSet::enqueue_completed_buffer(BufferNode* cbn) { |
57955
18863bf3501f
8230332: G1DirtyCardQueueSet _notify_when_complete is always true
kbarrett
parents:
57953
diff
changeset
|
111 |
MonitorLocker ml(_cbl_mon, Mutex::_no_safepoint_check_flag); |
55498 | 112 |
cbn->set_next(NULL); |
113 |
if (_completed_buffers_tail == NULL) { |
|
114 |
assert(_completed_buffers_head == NULL, "Well-formedness"); |
|
115 |
_completed_buffers_head = cbn; |
|
116 |
_completed_buffers_tail = cbn; |
|
117 |
} else { |
|
118 |
_completed_buffers_tail->set_next(cbn); |
|
119 |
_completed_buffers_tail = cbn; |
|
120 |
} |
|
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
121 |
_num_cards += buffer_size() - cbn->index(); |
55498 | 122 |
|
123 |
if (!process_completed_buffers() && |
|
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
124 |
(num_cards() > process_cards_threshold())) { |
55498 | 125 |
set_process_completed_buffers(true); |
57955
18863bf3501f
8230332: G1DirtyCardQueueSet _notify_when_complete is always true
kbarrett
parents:
57953
diff
changeset
|
126 |
ml.notify_all(); |
55498 | 127 |
} |
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
128 |
verify_num_cards(); |
55498 | 129 |
} |
130 |
||
131 |
BufferNode* G1DirtyCardQueueSet::get_completed_buffer(size_t stop_at) { |
|
132 |
MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag); |
|
133 |
||
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
134 |
if (num_cards() <= stop_at) { |
55498 | 135 |
return NULL; |
136 |
} |
|
137 |
||
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
138 |
assert(num_cards() > 0, "invariant"); |
55498 | 139 |
assert(_completed_buffers_head != NULL, "invariant"); |
140 |
assert(_completed_buffers_tail != NULL, "invariant"); |
|
141 |
||
142 |
BufferNode* bn = _completed_buffers_head; |
|
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
143 |
_num_cards -= buffer_size() - bn->index(); |
55498 | 144 |
_completed_buffers_head = bn->next(); |
145 |
if (_completed_buffers_head == NULL) { |
|
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
146 |
assert(num_cards() == 0, "invariant"); |
55498 | 147 |
_completed_buffers_tail = NULL; |
148 |
set_process_completed_buffers(false); |
|
149 |
} |
|
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
150 |
verify_num_cards(); |
55498 | 151 |
bn->set_next(NULL); |
152 |
return bn; |
|
153 |
} |
|
154 |
||
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
155 |
#ifdef ASSERT |
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
156 |
void G1DirtyCardQueueSet::verify_num_cards() const { |
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
157 |
size_t actual = 0; |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
158 |
BufferNode* cur = _completed_buffers_head; |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
159 |
while (cur != NULL) { |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
160 |
actual += buffer_size() - cur->index(); |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
161 |
cur = cur->next(); |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
162 |
} |
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
163 |
assert(actual == _num_cards, |
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
164 |
"Num entries in completed buffers should be " SIZE_FORMAT " but are " SIZE_FORMAT, |
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
165 |
_num_cards, actual); |
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
166 |
} |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
167 |
#endif |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
168 |
|
55498 | 169 |
void G1DirtyCardQueueSet::abandon_completed_buffers() { |
170 |
BufferNode* buffers_to_delete = NULL; |
|
171 |
{ |
|
172 |
MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag); |
|
173 |
buffers_to_delete = _completed_buffers_head; |
|
174 |
_completed_buffers_head = NULL; |
|
175 |
_completed_buffers_tail = NULL; |
|
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
176 |
_num_cards = 0; |
55498 | 177 |
set_process_completed_buffers(false); |
178 |
} |
|
179 |
while (buffers_to_delete != NULL) { |
|
180 |
BufferNode* bn = buffers_to_delete; |
|
181 |
buffers_to_delete = bn->next(); |
|
182 |
bn->set_next(NULL); |
|
183 |
deallocate_buffer(bn); |
|
184 |
} |
|
185 |
} |
|
186 |
||
187 |
void G1DirtyCardQueueSet::notify_if_necessary() { |
|
57955
18863bf3501f
8230332: G1DirtyCardQueueSet _notify_when_complete is always true
kbarrett
parents:
57953
diff
changeset
|
188 |
MonitorLocker ml(_cbl_mon, Mutex::_no_safepoint_check_flag); |
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
189 |
if (num_cards() > process_cards_threshold()) { |
55498 | 190 |
set_process_completed_buffers(true); |
57955
18863bf3501f
8230332: G1DirtyCardQueueSet _notify_when_complete is always true
kbarrett
parents:
57953
diff
changeset
|
191 |
ml.notify_all(); |
55498 | 192 |
} |
193 |
} |
|
194 |
||
195 |
// Merge lists of buffers. Notify the processing threads. |
|
196 |
// The source queue is emptied as a result. The queues |
|
197 |
// must share the monitor. |
|
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
198 |
void G1DirtyCardQueueSet::merge_bufferlists(G1RedirtyCardsQueueSet* src) { |
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
199 |
assert(allocator() == src->allocator(), "precondition"); |
57890
6bb824c45df1
8230192: Rename G1RedirtyCardsBufferList to G1BufferNodeList
kbarrett
parents:
57507
diff
changeset
|
200 |
const G1BufferNodeList from = src->take_all_completed_buffers(); |
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
201 |
if (from._head == NULL) return; |
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
202 |
|
55498 | 203 |
MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag); |
204 |
if (_completed_buffers_tail == NULL) { |
|
205 |
assert(_completed_buffers_head == NULL, "Well-formedness"); |
|
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
206 |
_completed_buffers_head = from._head; |
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
207 |
_completed_buffers_tail = from._tail; |
55498 | 208 |
} else { |
209 |
assert(_completed_buffers_head != NULL, "Well formedness"); |
|
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
210 |
_completed_buffers_tail->set_next(from._head); |
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
211 |
_completed_buffers_tail = from._tail; |
55498 | 212 |
} |
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
213 |
_num_cards += from._entry_count; |
55498 | 214 |
|
215 |
assert(_completed_buffers_head == NULL && _completed_buffers_tail == NULL || |
|
216 |
_completed_buffers_head != NULL && _completed_buffers_tail != NULL, |
|
217 |
"Sanity"); |
|
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
218 |
verify_num_cards(); |
55498 | 219 |
} |
220 |
||
58033
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
221 |
G1BufferNodeList G1DirtyCardQueueSet::take_all_completed_buffers() { |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
222 |
MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag); |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
223 |
G1BufferNodeList result(_completed_buffers_head, _completed_buffers_tail, _num_cards); |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
224 |
_completed_buffers_head = NULL; |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
225 |
_completed_buffers_tail = NULL; |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
226 |
_num_cards = 0; |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
227 |
return result; |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
228 |
} |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
229 |
|
59233 | 230 |
class G1RefineBufferedCards : public StackObj { |
231 |
BufferNode* const _node; |
|
232 |
CardTable::CardValue** const _node_buffer; |
|
233 |
const size_t _node_buffer_size; |
|
234 |
const uint _worker_id; |
|
235 |
size_t* _total_refined_cards; |
|
236 |
G1RemSet* const _g1rs; |
|
237 |
||
238 |
static inline int compare_card(const CardTable::CardValue* p1, |
|
239 |
const CardTable::CardValue* p2) { |
|
240 |
return p2 - p1; |
|
241 |
} |
|
242 |
||
243 |
// Sorts the cards from start_index to _node_buffer_size in *decreasing* |
|
244 |
// address order. Tests showed that this order is preferable to not sorting |
|
245 |
// or increasing address order. |
|
246 |
void sort_cards(size_t start_index) { |
|
247 |
QuickSort::sort(&_node_buffer[start_index], |
|
248 |
_node_buffer_size - start_index, |
|
249 |
compare_card, |
|
250 |
false); |
|
251 |
} |
|
252 |
||
253 |
// Returns the index to the first clean card in the buffer. |
|
254 |
size_t clean_cards() { |
|
255 |
const size_t start = _node->index(); |
|
256 |
assert(start <= _node_buffer_size, "invariant"); |
|
257 |
||
258 |
// Two-fingered compaction algorithm similar to the filtering mechanism in |
|
259 |
// SATBMarkQueue. The main difference is that clean_card_before_refine() |
|
260 |
// could change the buffer element in-place. |
|
261 |
// We don't check for SuspendibleThreadSet::should_yield(), because |
|
262 |
// cleaning and redirtying the cards is fast. |
|
263 |
CardTable::CardValue** src = &_node_buffer[start]; |
|
264 |
CardTable::CardValue** dst = &_node_buffer[_node_buffer_size]; |
|
265 |
assert(src <= dst, "invariant"); |
|
266 |
for ( ; src < dst; ++src) { |
|
267 |
// Search low to high for a card to keep. |
|
268 |
if (_g1rs->clean_card_before_refine(src)) { |
|
269 |
// Found keeper. Search high to low for a card to discard. |
|
270 |
while (src < --dst) { |
|
271 |
if (!_g1rs->clean_card_before_refine(dst)) { |
|
272 |
*dst = *src; // Replace discard with keeper. |
|
273 |
break; |
|
274 |
} |
|
275 |
} |
|
276 |
// If discard search failed (src == dst), the outer loop will also end. |
|
277 |
} |
|
278 |
} |
|
279 |
||
280 |
// dst points to the first retained clean card, or the end of the buffer |
|
281 |
// if all the cards were discarded. |
|
282 |
const size_t first_clean = dst - _node_buffer; |
|
283 |
assert(first_clean >= start && first_clean <= _node_buffer_size, "invariant"); |
|
284 |
// Discarded cards are considered as refined. |
|
285 |
*_total_refined_cards += first_clean - start; |
|
286 |
return first_clean; |
|
287 |
} |
|
288 |
||
289 |
bool refine_cleaned_cards(size_t start_index) { |
|
290 |
bool result = true; |
|
291 |
size_t i = start_index; |
|
292 |
for ( ; i < _node_buffer_size; ++i) { |
|
293 |
if (SuspendibleThreadSet::should_yield()) { |
|
294 |
redirty_unrefined_cards(i); |
|
295 |
result = false; |
|
296 |
break; |
|
297 |
} |
|
298 |
_g1rs->refine_card_concurrently(_node_buffer[i], _worker_id); |
|
299 |
} |
|
300 |
_node->set_index(i); |
|
301 |
*_total_refined_cards += i - start_index; |
|
302 |
return result; |
|
303 |
} |
|
304 |
||
305 |
void redirty_unrefined_cards(size_t start) { |
|
306 |
for ( ; start < _node_buffer_size; ++start) { |
|
307 |
*_node_buffer[start] = G1CardTable::dirty_card_val(); |
|
308 |
} |
|
309 |
} |
|
310 |
||
311 |
public: |
|
312 |
G1RefineBufferedCards(BufferNode* node, |
|
313 |
size_t node_buffer_size, |
|
314 |
uint worker_id, |
|
315 |
size_t* total_refined_cards) : |
|
316 |
_node(node), |
|
317 |
_node_buffer(reinterpret_cast<CardTable::CardValue**>(BufferNode::make_buffer_from_node(node))), |
|
318 |
_node_buffer_size(node_buffer_size), |
|
319 |
_worker_id(worker_id), |
|
320 |
_total_refined_cards(total_refined_cards), |
|
321 |
_g1rs(G1CollectedHeap::heap()->rem_set()) {} |
|
322 |
||
323 |
bool refine() { |
|
324 |
size_t first_clean_index = clean_cards(); |
|
325 |
if (first_clean_index == _node_buffer_size) { |
|
326 |
_node->set_index(first_clean_index); |
|
327 |
return true; |
|
328 |
} |
|
329 |
// This fence serves two purposes. First, the cards must be cleaned |
|
330 |
// before processing the contents. Second, we can't proceed with |
|
331 |
// processing a region until after the read of the region's top in |
|
332 |
// collect_and_clean_cards(), for synchronization with possibly concurrent |
|
333 |
// humongous object allocation (see comment at the StoreStore fence before |
|
334 |
// setting the regions' tops in humongous allocation path). |
|
335 |
// It's okay that reading region's top and reading region's type were racy |
|
336 |
// wrto each other. We need both set, in any order, to proceed. |
|
337 |
OrderAccess::fence(); |
|
338 |
sort_cards(first_clean_index); |
|
339 |
return refine_cleaned_cards(first_clean_index); |
|
340 |
} |
|
341 |
}; |
|
342 |
||
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
343 |
bool G1DirtyCardQueueSet::refine_buffer(BufferNode* node, |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
344 |
uint worker_id, |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
345 |
size_t* total_refined_cards) { |
59233 | 346 |
G1RefineBufferedCards buffered_cards(node, |
347 |
buffer_size(), |
|
348 |
worker_id, |
|
349 |
total_refined_cards); |
|
350 |
return buffered_cards.refine(); |
|
37112
fe72d1d082ff
8151781: DirtyCardQueue::apply_closure is unused
kbarrett
parents:
37065
diff
changeset
|
351 |
} |
fe72d1d082ff
8151781: DirtyCardQueue::apply_closure is unused
kbarrett
parents:
37065
diff
changeset
|
352 |
|
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
353 |
#ifndef ASSERT |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
354 |
#define assert_fully_consumed(node, buffer_size) |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
355 |
#else |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
356 |
#define assert_fully_consumed(node, buffer_size) \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
357 |
do { \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
358 |
size_t _afc_index = (node)->index(); \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
359 |
size_t _afc_size = (buffer_size); \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
360 |
assert(_afc_index == _afc_size, \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
361 |
"Buffer was not fully consumed as claimed: index: " \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
362 |
SIZE_FORMAT ", size: " SIZE_FORMAT, \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
363 |
_afc_index, _afc_size); \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
364 |
} while (0) |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
365 |
#endif // ASSERT |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
366 |
|
54970
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
367 |
bool G1DirtyCardQueueSet::process_or_enqueue_completed_buffer(BufferNode* node) { |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
368 |
if (Thread::current()->is_Java_thread()) { |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
369 |
// If the number of buffers exceeds the limit, make this Java |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
370 |
// thread do the processing itself. We don't lock to access |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
371 |
// buffer count or padding; it is fine to be imprecise here. The |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
372 |
// add of padding could overflow, which is treated as unlimited. |
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
373 |
size_t limit = max_cards() + max_cards_padding(); |
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
374 |
if ((num_cards() > limit) && (limit >= max_cards())) { |
54970
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
375 |
if (mut_process_buffer(node)) { |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
376 |
return true; |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
377 |
} |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
378 |
} |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
379 |
} |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
380 |
enqueue_completed_buffer(node); |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
381 |
return false; |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
382 |
} |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
383 |
|
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
384 |
bool G1DirtyCardQueueSet::mut_process_buffer(BufferNode* node) { |
57969
6f0215981777
8230327: Make G1DirtyCardQueueSet free-id init unconditional
kbarrett
parents:
57955
diff
changeset
|
385 |
uint worker_id = _free_ids.claim_par_id(); // temporarily claim an id |
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
386 |
uint counter_index = worker_id - par_ids_start(); |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
387 |
size_t* counter = &_mutator_refined_cards_counters[counter_index]; |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
388 |
bool result = refine_buffer(node, worker_id, counter); |
57969
6f0215981777
8230327: Make G1DirtyCardQueueSet free-id init unconditional
kbarrett
parents:
57955
diff
changeset
|
389 |
_free_ids.release_par_id(worker_id); // release the id |
1374 | 390 |
|
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
391 |
if (result) { |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
392 |
assert_fully_consumed(node, buffer_size()); |
1374 | 393 |
} |
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
394 |
return result; |
1374 | 395 |
} |
396 |
||
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
397 |
bool G1DirtyCardQueueSet::refine_completed_buffer_concurrently(uint worker_id, |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
398 |
size_t stop_at, |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
399 |
size_t* total_refined_cards) { |
58033
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
400 |
BufferNode* node = get_completed_buffer(stop_at); |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
401 |
if (node == NULL) { |
36179
03a0d7b8450f
8149793: DirtyCardQueueSet::apply_closure_to_completed_buffer_helper isn't helpful
kbarrett
parents:
35902
diff
changeset
|
402 |
return false; |
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
403 |
} else if (refine_buffer(node, worker_id, total_refined_cards)) { |
58033
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
404 |
assert_fully_consumed(node, buffer_size()); |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
405 |
// Done with fully processed buffer. |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
406 |
deallocate_buffer(node); |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
407 |
return true; |
36179
03a0d7b8450f
8149793: DirtyCardQueueSet::apply_closure_to_completed_buffer_helper isn't helpful
kbarrett
parents:
35902
diff
changeset
|
408 |
} else { |
58033
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
409 |
// Return partially processed buffer to the queue. |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
410 |
enqueue_completed_buffer(node); |
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
411 |
return true; |
36179
03a0d7b8450f
8149793: DirtyCardQueueSet::apply_closure_to_completed_buffer_helper isn't helpful
kbarrett
parents:
35902
diff
changeset
|
412 |
} |
6247 | 413 |
} |
414 |
||
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
415 |
void G1DirtyCardQueueSet::abandon_logs() { |
6247 | 416 |
assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); |
53102
35530ca3e0b2
8214201: Make PtrQueueSet completed buffer list private
kbarrett
parents:
53034
diff
changeset
|
417 |
abandon_completed_buffers(); |
54006 | 418 |
|
1374 | 419 |
// Since abandon is done only at safepoints, we can safely manipulate |
420 |
// these queues. |
|
54006 | 421 |
struct AbandonThreadLogClosure : public ThreadClosure { |
422 |
virtual void do_thread(Thread* t) { |
|
423 |
G1ThreadLocalData::dirty_card_queue(t).reset(); |
|
424 |
} |
|
425 |
} closure; |
|
426 |
Threads::threads_do(&closure); |
|
427 |
||
54255 | 428 |
G1BarrierSet::shared_dirty_card_queue().reset(); |
37065 | 429 |
} |
1374 | 430 |
|
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
431 |
void G1DirtyCardQueueSet::concatenate_logs() { |
1374 | 432 |
// Iterate over all the threads, if we find a partial log add it to |
433 |
// the global list of logs. Temporarily turn off the limit on the number |
|
434 |
// of outstanding buffers. |
|
435 |
assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); |
|
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
436 |
size_t old_limit = max_cards(); |
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
437 |
set_max_cards(MaxCardsUnlimited); |
54006 | 438 |
|
54255 | 439 |
struct ConcatenateThreadLogClosure : public ThreadClosure { |
54006 | 440 |
virtual void do_thread(Thread* t) { |
54255 | 441 |
G1DirtyCardQueue& dcq = G1ThreadLocalData::dirty_card_queue(t); |
442 |
if (!dcq.is_empty()) { |
|
443 |
dcq.flush(); |
|
444 |
} |
|
54006 | 445 |
} |
54255 | 446 |
} closure; |
54006 | 447 |
Threads::threads_do(&closure); |
448 |
||
54255 | 449 |
G1BarrierSet::shared_dirty_card_queue().flush(); |
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
450 |
set_max_cards(old_limit); |
1374 | 451 |
} |