author | tschatzl |
Wed, 24 Jul 2019 11:49:39 +0200 | |
changeset 57507 | f6b30bd6804e |
parent 55752 | 8ae33203d600 |
child 57890 | 6bb824c45df1 |
permissions | -rw-r--r-- |
1374 | 1 |
/* |
53482 | 2 |
* Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved. |
1374 | 3 |
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
4 |
* |
|
5 |
* This code is free software; you can redistribute it and/or modify it |
|
6 |
* under the terms of the GNU General Public License version 2 only, as |
|
7 |
* published by the Free Software Foundation. |
|
8 |
* |
|
9 |
* This code is distributed in the hope that it will be useful, but WITHOUT |
|
10 |
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|
11 |
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
|
12 |
* version 2 for more details (a copy is included in the LICENSE file that |
|
13 |
* accompanied this code). |
|
14 |
* |
|
15 |
* You should have received a copy of the GNU General Public License version |
|
16 |
* 2 along with this work; if not, write to the Free Software Foundation, |
|
17 |
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
|
18 |
* |
|
5547
f4b087cbb361
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
4481
diff
changeset
|
19 |
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
f4b087cbb361
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
4481
diff
changeset
|
20 |
* or visit www.oracle.com if you need additional information or have any |
f4b087cbb361
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
4481
diff
changeset
|
21 |
* questions. |
1374 | 22 |
* |
23 |
*/ |
|
24 |
||
7397 | 25 |
#include "precompiled.hpp" |
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
26 |
#include "gc/g1/g1CardTableEntryClosure.hpp" |
30764 | 27 |
#include "gc/g1/g1CollectedHeap.inline.hpp" |
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
28 |
#include "gc/g1/g1DirtyCardQueue.hpp" |
53482 | 29 |
#include "gc/g1/g1FreeIdSet.hpp" |
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
30 |
#include "gc/g1/g1RedirtyCardsQueue.hpp" |
46653 | 31 |
#include "gc/g1/g1RemSet.hpp" |
49752
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49333
diff
changeset
|
32 |
#include "gc/g1/g1ThreadLocalData.hpp" |
30764 | 33 |
#include "gc/g1/heapRegionRemSet.hpp" |
52576
367ca5f32505
8213829: Remove circular dependency between g1CollectedHeap and g1ConcurrentMark
manc
parents:
49752
diff
changeset
|
34 |
#include "gc/shared/suspendibleThreadSet.hpp" |
30764 | 35 |
#include "gc/shared/workgroup.hpp" |
40655
9f644073d3a0
8157907: Incorrect inclusion of atomic.hpp instead of atomic.inline.hpp
dholmes
parents:
40012
diff
changeset
|
36 |
#include "runtime/atomic.hpp" |
52726
9cfa2e273b77
8214278: Cleanup process_completed_threshold and related state
kbarrett
parents:
52684
diff
changeset
|
37 |
#include "runtime/flags/flagSetting.hpp" |
7397 | 38 |
#include "runtime/mutexLocker.hpp" |
39 |
#include "runtime/safepoint.hpp" |
|
14583
d70ee55535f4
8003935: Simplify the needed includes for using Thread::current()
stefank
parents:
10565
diff
changeset
|
40 |
#include "runtime/thread.inline.hpp" |
48105
8d15b1369c7a
8167108: inconsistent handling of SR_lock can lead to crashes
dcubed
parents:
47634
diff
changeset
|
41 |
#include "runtime/threadSMR.hpp" |
1374 | 42 |
|
46653 | 43 |
// Closure used for updating remembered sets and recording references that |
44 |
// point into the collection set while the mutator is running. |
|
45 |
// Assumed to be only executed concurrently with the mutator. Yields via |
|
46 |
// SuspendibleThreadSet after every card. |
|
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
47 |
class G1RefineCardConcurrentlyClosure: public G1CardTableEntryClosure { |
46653 | 48 |
public: |
54110 | 49 |
bool do_card_ptr(CardValue* card_ptr, uint worker_i) { |
53985
0da5c17f5e43
8219747: Remove g1_ prefix to g1_remset and g1_policy members in G1CollectedHeap
tschatzl
parents:
53747
diff
changeset
|
50 |
G1CollectedHeap::heap()->rem_set()->refine_card_concurrently(card_ptr, worker_i); |
46653 | 51 |
|
52 |
if (SuspendibleThreadSet::should_yield()) { |
|
53 |
// Caller will actually yield. |
|
54 |
return false; |
|
55 |
} |
|
56 |
// Otherwise, we finished successfully; return true. |
|
57 |
return true; |
|
58 |
} |
|
59 |
}; |
|
60 |
||
54255 | 61 |
G1DirtyCardQueue::G1DirtyCardQueue(G1DirtyCardQueueSet* qset) : |
33761
329db4b51480
6899049: G1: Clean up code in ptrQueue.[ch]pp and ptrQueue.inline.hpp
kbarrett
parents:
30764
diff
changeset
|
62 |
// Dirty card queues are always active, so we create them with their |
329db4b51480
6899049: G1: Clean up code in ptrQueue.[ch]pp and ptrQueue.inline.hpp
kbarrett
parents:
30764
diff
changeset
|
63 |
// active field set to true. |
54255 | 64 |
PtrQueue(qset, true /* active */) |
33761
329db4b51480
6899049: G1: Clean up code in ptrQueue.[ch]pp and ptrQueue.inline.hpp
kbarrett
parents:
30764
diff
changeset
|
65 |
{ } |
329db4b51480
6899049: G1: Clean up code in ptrQueue.[ch]pp and ptrQueue.inline.hpp
kbarrett
parents:
30764
diff
changeset
|
66 |
|
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
67 |
G1DirtyCardQueue::~G1DirtyCardQueue() { |
54255 | 68 |
flush(); |
33761
329db4b51480
6899049: G1: Clean up code in ptrQueue.[ch]pp and ptrQueue.inline.hpp
kbarrett
parents:
30764
diff
changeset
|
69 |
} |
329db4b51480
6899049: G1: Clean up code in ptrQueue.[ch]pp and ptrQueue.inline.hpp
kbarrett
parents:
30764
diff
changeset
|
70 |
|
54970
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
71 |
void G1DirtyCardQueue::handle_completed_buffer() { |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
72 |
assert(_buf != NULL, "precondition"); |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
73 |
BufferNode* node = BufferNode::make_node_from_buffer(_buf, index()); |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
74 |
G1DirtyCardQueueSet* dcqs = dirty_card_qset(); |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
75 |
if (dcqs->process_or_enqueue_completed_buffer(node)) { |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
76 |
reset(); // Buffer fully processed, reset index. |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
77 |
} else { |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
78 |
allocate_buffer(); // Buffer enqueued, get a new one. |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
79 |
} |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
80 |
} |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
81 |
|
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
82 |
G1DirtyCardQueueSet::G1DirtyCardQueueSet(bool notify_when_complete) : |
55498 | 83 |
PtrQueueSet(), |
84 |
_cbl_mon(NULL), |
|
85 |
_completed_buffers_head(NULL), |
|
86 |
_completed_buffers_tail(NULL), |
|
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
87 |
_num_entries_in_completed_buffers(0), |
55498 | 88 |
_process_completed_buffers_threshold(ProcessCompletedBuffersThresholdNever), |
89 |
_process_completed_buffers(false), |
|
90 |
_notify_when_complete(notify_when_complete), |
|
54970
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
91 |
_max_completed_buffers(MaxCompletedBuffersUnlimited), |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
92 |
_completed_buffers_padding(0), |
1374 | 93 |
_free_ids(NULL), |
53034
de99beff5c0e
8214946: G1: Initialize all class members on construction
lkorinth
parents:
52726
diff
changeset
|
94 |
_processed_buffers_mut(0), |
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
95 |
_processed_buffers_rs_thread(0) |
1374 | 96 |
{ |
97 |
_all_active = true; |
|
98 |
} |
|
99 |
||
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
100 |
G1DirtyCardQueueSet::~G1DirtyCardQueueSet() { |
55498 | 101 |
abandon_completed_buffers(); |
53482 | 102 |
delete _free_ids; |
103 |
} |
|
104 |
||
2882
d508a8bac491
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
2881
diff
changeset
|
105 |
// Determines how many mutator threads can process the buffers in parallel. |
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
106 |
uint G1DirtyCardQueueSet::num_par_ids() { |
40012
f69cfe79fe98
8161993: G1 crashes if active_processor_count changes during startup
tschatzl
parents:
37197
diff
changeset
|
107 |
return (uint)os::initial_active_processor_count(); |
1374 | 108 |
} |
109 |
||
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
110 |
void G1DirtyCardQueueSet::initialize(Monitor* cbl_mon, |
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
111 |
BufferNode::Allocator* allocator, |
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
112 |
bool init_free_ids) { |
55498 | 113 |
PtrQueueSet::initialize(allocator); |
114 |
assert(_cbl_mon == NULL, "Init order issue?"); |
|
115 |
_cbl_mon = cbl_mon; |
|
35465 | 116 |
if (init_free_ids) { |
53482 | 117 |
_free_ids = new G1FreeIdSet(0, num_par_ids()); |
35465 | 118 |
} |
1374 | 119 |
} |
120 |
||
54006 | 121 |
void G1DirtyCardQueueSet::handle_zero_index_for_thread(Thread* t) { |
49752
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49333
diff
changeset
|
122 |
G1ThreadLocalData::dirty_card_queue(t).handle_zero_index(); |
1374 | 123 |
} |
124 |
||
55498 | 125 |
void G1DirtyCardQueueSet::enqueue_completed_buffer(BufferNode* cbn) { |
126 |
MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag); |
|
127 |
cbn->set_next(NULL); |
|
128 |
if (_completed_buffers_tail == NULL) { |
|
129 |
assert(_completed_buffers_head == NULL, "Well-formedness"); |
|
130 |
_completed_buffers_head = cbn; |
|
131 |
_completed_buffers_tail = cbn; |
|
132 |
} else { |
|
133 |
_completed_buffers_tail->set_next(cbn); |
|
134 |
_completed_buffers_tail = cbn; |
|
135 |
} |
|
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
136 |
_num_entries_in_completed_buffers += buffer_size() - cbn->index(); |
55498 | 137 |
|
138 |
if (!process_completed_buffers() && |
|
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
139 |
(num_completed_buffers() > process_completed_buffers_threshold())) { |
55498 | 140 |
set_process_completed_buffers(true); |
141 |
if (_notify_when_complete) { |
|
142 |
_cbl_mon->notify_all(); |
|
143 |
} |
|
144 |
} |
|
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
145 |
verify_num_entries_in_completed_buffers(); |
55498 | 146 |
} |
147 |
||
148 |
BufferNode* G1DirtyCardQueueSet::get_completed_buffer(size_t stop_at) { |
|
149 |
MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag); |
|
150 |
||
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
151 |
if (num_completed_buffers() <= stop_at) { |
55498 | 152 |
return NULL; |
153 |
} |
|
154 |
||
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
155 |
assert(num_completed_buffers() > 0, "invariant"); |
55498 | 156 |
assert(_completed_buffers_head != NULL, "invariant"); |
157 |
assert(_completed_buffers_tail != NULL, "invariant"); |
|
158 |
||
159 |
BufferNode* bn = _completed_buffers_head; |
|
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
160 |
_num_entries_in_completed_buffers -= buffer_size() - bn->index(); |
55498 | 161 |
_completed_buffers_head = bn->next(); |
162 |
if (_completed_buffers_head == NULL) { |
|
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
163 |
assert(num_completed_buffers() == 0, "invariant"); |
55498 | 164 |
_completed_buffers_tail = NULL; |
165 |
set_process_completed_buffers(false); |
|
166 |
} |
|
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
167 |
verify_num_entries_in_completed_buffers(); |
55498 | 168 |
bn->set_next(NULL); |
169 |
return bn; |
|
170 |
} |
|
171 |
||
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
172 |
#ifdef ASSERT |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
173 |
void G1DirtyCardQueueSet::verify_num_entries_in_completed_buffers() const { |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
174 |
size_t actual = 0; |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
175 |
BufferNode* cur = _completed_buffers_head; |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
176 |
while (cur != NULL) { |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
177 |
actual += buffer_size() - cur->index(); |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
178 |
cur = cur->next(); |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
179 |
} |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
180 |
assert(actual == _num_entries_in_completed_buffers, |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
181 |
"Num entries in completed buffers should be " SIZE_FORMAT " but are " SIZE_FORMAT, |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
182 |
_num_entries_in_completed_buffers, actual); |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
183 |
} |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
184 |
#endif |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
185 |
|
55498 | 186 |
void G1DirtyCardQueueSet::abandon_completed_buffers() { |
187 |
BufferNode* buffers_to_delete = NULL; |
|
188 |
{ |
|
189 |
MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag); |
|
190 |
buffers_to_delete = _completed_buffers_head; |
|
191 |
_completed_buffers_head = NULL; |
|
192 |
_completed_buffers_tail = NULL; |
|
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
193 |
_num_entries_in_completed_buffers = 0; |
55498 | 194 |
set_process_completed_buffers(false); |
195 |
} |
|
196 |
while (buffers_to_delete != NULL) { |
|
197 |
BufferNode* bn = buffers_to_delete; |
|
198 |
buffers_to_delete = bn->next(); |
|
199 |
bn->set_next(NULL); |
|
200 |
deallocate_buffer(bn); |
|
201 |
} |
|
202 |
} |
|
203 |
||
204 |
void G1DirtyCardQueueSet::notify_if_necessary() { |
|
205 |
MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag); |
|
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
206 |
if (num_completed_buffers() > process_completed_buffers_threshold()) { |
55498 | 207 |
set_process_completed_buffers(true); |
208 |
if (_notify_when_complete) |
|
209 |
_cbl_mon->notify(); |
|
210 |
} |
|
211 |
} |
|
212 |
||
213 |
// Merge lists of buffers. Notify the processing threads. |
|
214 |
// The source queue is emptied as a result. The queues |
|
215 |
// must share the monitor. |
|
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
216 |
void G1DirtyCardQueueSet::merge_bufferlists(G1RedirtyCardsQueueSet* src) { |
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
217 |
assert(allocator() == src->allocator(), "precondition"); |
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
218 |
const G1RedirtyCardsBufferList from = src->take_all_completed_buffers(); |
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
219 |
if (from._head == NULL) return; |
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
220 |
|
55498 | 221 |
MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag); |
222 |
if (_completed_buffers_tail == NULL) { |
|
223 |
assert(_completed_buffers_head == NULL, "Well-formedness"); |
|
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
224 |
_completed_buffers_head = from._head; |
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
225 |
_completed_buffers_tail = from._tail; |
55498 | 226 |
} else { |
227 |
assert(_completed_buffers_head != NULL, "Well formedness"); |
|
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
228 |
_completed_buffers_tail->set_next(from._head); |
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
229 |
_completed_buffers_tail = from._tail; |
55498 | 230 |
} |
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
231 |
_num_entries_in_completed_buffers += from._entry_count; |
55498 | 232 |
|
233 |
assert(_completed_buffers_head == NULL && _completed_buffers_tail == NULL || |
|
234 |
_completed_buffers_head != NULL && _completed_buffers_tail != NULL, |
|
235 |
"Sanity"); |
|
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
236 |
verify_num_entries_in_completed_buffers(); |
55498 | 237 |
} |
238 |
||
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
239 |
bool G1DirtyCardQueueSet::apply_closure_to_buffer(G1CardTableEntryClosure* cl, |
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
240 |
BufferNode* node, |
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
241 |
uint worker_i) { |
37112
fe72d1d082ff
8151781: DirtyCardQueue::apply_closure is unused
kbarrett
parents:
37065
diff
changeset
|
242 |
if (cl == NULL) return true; |
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
243 |
bool result = true; |
37112
fe72d1d082ff
8151781: DirtyCardQueue::apply_closure is unused
kbarrett
parents:
37065
diff
changeset
|
244 |
void** buf = BufferNode::make_buffer_from_node(node); |
46443 | 245 |
size_t i = node->index(); |
246 |
size_t limit = buffer_size(); |
|
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
247 |
for ( ; i < limit; ++i) { |
54110 | 248 |
CardTable::CardValue* card_ptr = static_cast<CardTable::CardValue*>(buf[i]); |
37112
fe72d1d082ff
8151781: DirtyCardQueue::apply_closure is unused
kbarrett
parents:
37065
diff
changeset
|
249 |
assert(card_ptr != NULL, "invariant"); |
fe72d1d082ff
8151781: DirtyCardQueue::apply_closure is unused
kbarrett
parents:
37065
diff
changeset
|
250 |
if (!cl->do_card_ptr(card_ptr, worker_i)) { |
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
251 |
result = false; // Incomplete processing. |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
252 |
break; |
37112
fe72d1d082ff
8151781: DirtyCardQueue::apply_closure is unused
kbarrett
parents:
37065
diff
changeset
|
253 |
} |
fe72d1d082ff
8151781: DirtyCardQueue::apply_closure is unused
kbarrett
parents:
37065
diff
changeset
|
254 |
} |
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
255 |
assert(i <= buffer_size(), "invariant"); |
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
256 |
node->set_index(i); |
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
257 |
return result; |
37112
fe72d1d082ff
8151781: DirtyCardQueue::apply_closure is unused
kbarrett
parents:
37065
diff
changeset
|
258 |
} |
fe72d1d082ff
8151781: DirtyCardQueue::apply_closure is unused
kbarrett
parents:
37065
diff
changeset
|
259 |
|
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
260 |
#ifndef ASSERT |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
261 |
#define assert_fully_consumed(node, buffer_size) |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
262 |
#else |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
263 |
#define assert_fully_consumed(node, buffer_size) \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
264 |
do { \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
265 |
size_t _afc_index = (node)->index(); \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
266 |
size_t _afc_size = (buffer_size); \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
267 |
assert(_afc_index == _afc_size, \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
268 |
"Buffer was not fully consumed as claimed: index: " \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
269 |
SIZE_FORMAT ", size: " SIZE_FORMAT, \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
270 |
_afc_index, _afc_size); \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
271 |
} while (0) |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
272 |
#endif // ASSERT |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
273 |
|
54970
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
274 |
bool G1DirtyCardQueueSet::process_or_enqueue_completed_buffer(BufferNode* node) { |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
275 |
if (Thread::current()->is_Java_thread()) { |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
276 |
// If the number of buffers exceeds the limit, make this Java |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
277 |
// thread do the processing itself. We don't lock to access |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
278 |
// buffer count or padding; it is fine to be imprecise here. The |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
279 |
// add of padding could overflow, which is treated as unlimited. |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
280 |
size_t max_buffers = max_completed_buffers(); |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
281 |
size_t limit = max_buffers + completed_buffers_padding(); |
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
282 |
if ((num_completed_buffers() > limit) && (limit >= max_buffers)) { |
54970
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
283 |
if (mut_process_buffer(node)) { |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
284 |
return true; |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
285 |
} |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
286 |
} |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
287 |
} |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
288 |
enqueue_completed_buffer(node); |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
289 |
return false; |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
290 |
} |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
291 |
|
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
292 |
bool G1DirtyCardQueueSet::mut_process_buffer(BufferNode* node) { |
35465 | 293 |
guarantee(_free_ids != NULL, "must be"); |
1374 | 294 |
|
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
295 |
uint worker_i = _free_ids->claim_par_id(); // temporarily claim an id |
46653 | 296 |
G1RefineCardConcurrentlyClosure cl; |
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
297 |
bool result = apply_closure_to_buffer(&cl, node, worker_i); |
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
298 |
_free_ids->release_par_id(worker_i); // release the id |
1374 | 299 |
|
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
300 |
if (result) { |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
301 |
assert_fully_consumed(node, buffer_size()); |
35481
53825bdb41a0
8145038: Simplify mut_process_buffer worker id management
aharlap
parents:
35465
diff
changeset
|
302 |
Atomic::inc(&_processed_buffers_mut); |
1374 | 303 |
} |
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
304 |
return result; |
1374 | 305 |
} |
306 |
||
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
307 |
bool G1DirtyCardQueueSet::refine_completed_buffer_concurrently(uint worker_i, size_t stop_at) { |
46653 | 308 |
G1RefineCardConcurrentlyClosure cl; |
309 |
return apply_closure_to_completed_buffer(&cl, worker_i, stop_at, false); |
|
310 |
} |
|
311 |
||
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
312 |
bool G1DirtyCardQueueSet::apply_closure_during_gc(G1CardTableEntryClosure* cl, uint worker_i) { |
49333
489f1dd40582
8196876: OopStorage::assert_at_safepoint clashes with assert_at_safepoint macros in g1CollectedHeap.hpp
kbarrett
parents:
48105
diff
changeset
|
313 |
assert_at_safepoint(); |
46653 | 314 |
return apply_closure_to_completed_buffer(cl, worker_i, 0, true); |
315 |
} |
|
316 |
||
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
317 |
bool G1DirtyCardQueueSet::apply_closure_to_completed_buffer(G1CardTableEntryClosure* cl, |
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
318 |
uint worker_i, |
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
319 |
size_t stop_at, |
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
320 |
bool during_pause) { |
6247 | 321 |
assert(!during_pause || stop_at == 0, "Should not leave any completed buffers during a pause"); |
322 |
BufferNode* nd = get_completed_buffer(stop_at); |
|
36179
03a0d7b8450f
8149793: DirtyCardQueueSet::apply_closure_to_completed_buffer_helper isn't helpful
kbarrett
parents:
35902
diff
changeset
|
323 |
if (nd == NULL) { |
03a0d7b8450f
8149793: DirtyCardQueueSet::apply_closure_to_completed_buffer_helper isn't helpful
kbarrett
parents:
35902
diff
changeset
|
324 |
return false; |
03a0d7b8450f
8149793: DirtyCardQueueSet::apply_closure_to_completed_buffer_helper isn't helpful
kbarrett
parents:
35902
diff
changeset
|
325 |
} else { |
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
326 |
if (apply_closure_to_buffer(cl, nd, worker_i)) { |
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
327 |
assert_fully_consumed(nd, buffer_size()); |
36179
03a0d7b8450f
8149793: DirtyCardQueueSet::apply_closure_to_completed_buffer_helper isn't helpful
kbarrett
parents:
35902
diff
changeset
|
328 |
// Done with fully processed buffer. |
37065 | 329 |
deallocate_buffer(nd); |
36179
03a0d7b8450f
8149793: DirtyCardQueueSet::apply_closure_to_completed_buffer_helper isn't helpful
kbarrett
parents:
35902
diff
changeset
|
330 |
Atomic::inc(&_processed_buffers_rs_thread); |
03a0d7b8450f
8149793: DirtyCardQueueSet::apply_closure_to_completed_buffer_helper isn't helpful
kbarrett
parents:
35902
diff
changeset
|
331 |
} else { |
03a0d7b8450f
8149793: DirtyCardQueueSet::apply_closure_to_completed_buffer_helper isn't helpful
kbarrett
parents:
35902
diff
changeset
|
332 |
// Return partially processed buffer to the queue. |
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
333 |
guarantee(!during_pause, "Should never stop early"); |
53102
35530ca3e0b2
8214201: Make PtrQueueSet completed buffer list private
kbarrett
parents:
53034
diff
changeset
|
334 |
enqueue_completed_buffer(nd); |
36179
03a0d7b8450f
8149793: DirtyCardQueueSet::apply_closure_to_completed_buffer_helper isn't helpful
kbarrett
parents:
35902
diff
changeset
|
335 |
} |
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
336 |
return true; |
36179
03a0d7b8450f
8149793: DirtyCardQueueSet::apply_closure_to_completed_buffer_helper isn't helpful
kbarrett
parents:
35902
diff
changeset
|
337 |
} |
6247 | 338 |
} |
339 |
||
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
340 |
void G1DirtyCardQueueSet::abandon_logs() { |
6247 | 341 |
assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); |
53102
35530ca3e0b2
8214201: Make PtrQueueSet completed buffer list private
kbarrett
parents:
53034
diff
changeset
|
342 |
abandon_completed_buffers(); |
54006 | 343 |
|
1374 | 344 |
// Since abandon is done only at safepoints, we can safely manipulate |
345 |
// these queues. |
|
54006 | 346 |
struct AbandonThreadLogClosure : public ThreadClosure { |
347 |
virtual void do_thread(Thread* t) { |
|
348 |
G1ThreadLocalData::dirty_card_queue(t).reset(); |
|
349 |
} |
|
350 |
} closure; |
|
351 |
Threads::threads_do(&closure); |
|
352 |
||
54255 | 353 |
G1BarrierSet::shared_dirty_card_queue().reset(); |
37065 | 354 |
} |
1374 | 355 |
|
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
356 |
void G1DirtyCardQueueSet::concatenate_logs() { |
1374 | 357 |
// Iterate over all the threads, if we find a partial log add it to |
358 |
// the global list of logs. Temporarily turn off the limit on the number |
|
359 |
// of outstanding buffers. |
|
360 |
assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); |
|
53102
35530ca3e0b2
8214201: Make PtrQueueSet completed buffer list private
kbarrett
parents:
53034
diff
changeset
|
361 |
size_t old_limit = max_completed_buffers(); |
35530ca3e0b2
8214201: Make PtrQueueSet completed buffer list private
kbarrett
parents:
53034
diff
changeset
|
362 |
set_max_completed_buffers(MaxCompletedBuffersUnlimited); |
54006 | 363 |
|
54255 | 364 |
struct ConcatenateThreadLogClosure : public ThreadClosure { |
54006 | 365 |
virtual void do_thread(Thread* t) { |
54255 | 366 |
G1DirtyCardQueue& dcq = G1ThreadLocalData::dirty_card_queue(t); |
367 |
if (!dcq.is_empty()) { |
|
368 |
dcq.flush(); |
|
369 |
} |
|
54006 | 370 |
} |
54255 | 371 |
} closure; |
54006 | 372 |
Threads::threads_do(&closure); |
373 |
||
54255 | 374 |
G1BarrierSet::shared_dirty_card_queue().flush(); |
53102
35530ca3e0b2
8214201: Make PtrQueueSet completed buffer list private
kbarrett
parents:
53034
diff
changeset
|
375 |
set_max_completed_buffers(old_limit); |
1374 | 376 |
} |