author | chegar |
Thu, 17 Oct 2019 20:54:25 +0100 | |
branch | datagramsocketimpl-branch |
changeset 58679 | 9c3209ff7550 |
parent 58678 | 9cf78a70fa4f |
parent 58508 | d6058bd73982 |
permissions | -rw-r--r-- |
1374 | 1 |
/* |
53482 | 2 |
* Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved. |
1374 | 3 |
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
4 |
* |
|
5 |
* This code is free software; you can redistribute it and/or modify it |
|
6 |
* under the terms of the GNU General Public License version 2 only, as |
|
7 |
* published by the Free Software Foundation. |
|
8 |
* |
|
9 |
* This code is distributed in the hope that it will be useful, but WITHOUT |
|
10 |
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|
11 |
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
|
12 |
* version 2 for more details (a copy is included in the LICENSE file that |
|
13 |
* accompanied this code). |
|
14 |
* |
|
15 |
* You should have received a copy of the GNU General Public License version |
|
16 |
* 2 along with this work; if not, write to the Free Software Foundation, |
|
17 |
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
|
18 |
* |
|
5547
f4b087cbb361
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
4481
diff
changeset
|
19 |
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
f4b087cbb361
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
4481
diff
changeset
|
20 |
* or visit www.oracle.com if you need additional information or have any |
f4b087cbb361
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
4481
diff
changeset
|
21 |
* questions. |
1374 | 22 |
* |
23 |
*/ |
|
24 |
||
7397 | 25 |
#include "precompiled.hpp" |
57890
6bb824c45df1
8230192: Rename G1RedirtyCardsBufferList to G1BufferNodeList
kbarrett
parents:
57507
diff
changeset
|
26 |
#include "gc/g1/g1BufferNodeList.hpp" |
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
27 |
#include "gc/g1/g1CardTableEntryClosure.hpp" |
30764 | 28 |
#include "gc/g1/g1CollectedHeap.inline.hpp" |
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
29 |
#include "gc/g1/g1DirtyCardQueue.hpp" |
53482 | 30 |
#include "gc/g1/g1FreeIdSet.hpp" |
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
31 |
#include "gc/g1/g1RedirtyCardsQueue.hpp" |
46653 | 32 |
#include "gc/g1/g1RemSet.hpp" |
49752
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49333
diff
changeset
|
33 |
#include "gc/g1/g1ThreadLocalData.hpp" |
30764 | 34 |
#include "gc/g1/heapRegionRemSet.hpp" |
52576
367ca5f32505
8213829: Remove circular dependency between g1CollectedHeap and g1ConcurrentMark
manc
parents:
49752
diff
changeset
|
35 |
#include "gc/shared/suspendibleThreadSet.hpp" |
30764 | 36 |
#include "gc/shared/workgroup.hpp" |
40655
9f644073d3a0
8157907: Incorrect inclusion of atomic.hpp instead of atomic.inline.hpp
dholmes
parents:
40012
diff
changeset
|
37 |
#include "runtime/atomic.hpp" |
52726
9cfa2e273b77
8214278: Cleanup process_completed_threshold and related state
kbarrett
parents:
52684
diff
changeset
|
38 |
#include "runtime/flags/flagSetting.hpp" |
7397 | 39 |
#include "runtime/mutexLocker.hpp" |
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
40 |
#include "runtime/os.hpp" |
7397 | 41 |
#include "runtime/safepoint.hpp" |
14583
d70ee55535f4
8003935: Simplify the needed includes for using Thread::current()
stefank
parents:
10565
diff
changeset
|
42 |
#include "runtime/thread.inline.hpp" |
48105
8d15b1369c7a
8167108: inconsistent handling of SR_lock can lead to crashes
dcubed
parents:
47634
diff
changeset
|
43 |
#include "runtime/threadSMR.hpp" |
1374 | 44 |
|
54255 | 45 |
G1DirtyCardQueue::G1DirtyCardQueue(G1DirtyCardQueueSet* qset) : |
33761
329db4b51480
6899049: G1: Clean up code in ptrQueue.[ch]pp and ptrQueue.inline.hpp
kbarrett
parents:
30764
diff
changeset
|
46 |
// Dirty card queues are always active, so we create them with their |
329db4b51480
6899049: G1: Clean up code in ptrQueue.[ch]pp and ptrQueue.inline.hpp
kbarrett
parents:
30764
diff
changeset
|
47 |
// active field set to true. |
54255 | 48 |
PtrQueue(qset, true /* active */) |
33761
329db4b51480
6899049: G1: Clean up code in ptrQueue.[ch]pp and ptrQueue.inline.hpp
kbarrett
parents:
30764
diff
changeset
|
49 |
{ } |
329db4b51480
6899049: G1: Clean up code in ptrQueue.[ch]pp and ptrQueue.inline.hpp
kbarrett
parents:
30764
diff
changeset
|
50 |
|
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
51 |
G1DirtyCardQueue::~G1DirtyCardQueue() { |
54255 | 52 |
flush(); |
33761
329db4b51480
6899049: G1: Clean up code in ptrQueue.[ch]pp and ptrQueue.inline.hpp
kbarrett
parents:
30764
diff
changeset
|
53 |
} |
329db4b51480
6899049: G1: Clean up code in ptrQueue.[ch]pp and ptrQueue.inline.hpp
kbarrett
parents:
30764
diff
changeset
|
54 |
|
54970
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
55 |
void G1DirtyCardQueue::handle_completed_buffer() { |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
56 |
assert(_buf != NULL, "precondition"); |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
57 |
BufferNode* node = BufferNode::make_node_from_buffer(_buf, index()); |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
58 |
G1DirtyCardQueueSet* dcqs = dirty_card_qset(); |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
59 |
if (dcqs->process_or_enqueue_completed_buffer(node)) { |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
60 |
reset(); // Buffer fully processed, reset index. |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
61 |
} else { |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
62 |
allocate_buffer(); // Buffer enqueued, get a new one. |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
63 |
} |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
64 |
} |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
65 |
|
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
66 |
// Assumed to be zero by concurrent threads. |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
67 |
static uint par_ids_start() { return 0; } |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
68 |
|
58059
baa4dd528de0
8221361: Eliminate two-phase initialization for PtrQueueSet classes
kbarrett
parents:
58033
diff
changeset
|
69 |
G1DirtyCardQueueSet::G1DirtyCardQueueSet(Monitor* cbl_mon, |
baa4dd528de0
8221361: Eliminate two-phase initialization for PtrQueueSet classes
kbarrett
parents:
58033
diff
changeset
|
70 |
BufferNode::Allocator* allocator) : |
baa4dd528de0
8221361: Eliminate two-phase initialization for PtrQueueSet classes
kbarrett
parents:
58033
diff
changeset
|
71 |
PtrQueueSet(allocator), |
baa4dd528de0
8221361: Eliminate two-phase initialization for PtrQueueSet classes
kbarrett
parents:
58033
diff
changeset
|
72 |
_cbl_mon(cbl_mon), |
55498 | 73 |
_completed_buffers_head(NULL), |
74 |
_completed_buffers_tail(NULL), |
|
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
75 |
_num_cards(0), |
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
76 |
_process_cards_threshold(ProcessCardsThresholdNever), |
55498 | 77 |
_process_completed_buffers(false), |
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
78 |
_max_cards(MaxCardsUnlimited), |
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
79 |
_max_cards_padding(0), |
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
80 |
_free_ids(par_ids_start(), num_par_ids()), |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
81 |
_mutator_refined_cards_counters(NEW_C_HEAP_ARRAY(size_t, num_par_ids(), mtGC)) |
1374 | 82 |
{ |
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
83 |
::memset(_mutator_refined_cards_counters, 0, num_par_ids() * sizeof(size_t)); |
1374 | 84 |
_all_active = true; |
85 |
} |
|
86 |
||
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
87 |
G1DirtyCardQueueSet::~G1DirtyCardQueueSet() { |
55498 | 88 |
abandon_completed_buffers(); |
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
89 |
FREE_C_HEAP_ARRAY(size_t, _mutator_refined_cards_counters); |
53482 | 90 |
} |
91 |
||
2882
d508a8bac491
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
2881
diff
changeset
|
92 |
// Determines how many mutator threads can process the buffers in parallel. |
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
93 |
uint G1DirtyCardQueueSet::num_par_ids() { |
40012
f69cfe79fe98
8161993: G1 crashes if active_processor_count changes during startup
tschatzl
parents:
37197
diff
changeset
|
94 |
return (uint)os::initial_active_processor_count(); |
1374 | 95 |
} |
96 |
||
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
97 |
size_t G1DirtyCardQueueSet::total_mutator_refined_cards() const { |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
98 |
size_t sum = 0; |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
99 |
for (uint i = 0; i < num_par_ids(); ++i) { |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
100 |
sum += _mutator_refined_cards_counters[i]; |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
101 |
} |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
102 |
return sum; |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
103 |
} |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
104 |
|
54006 | 105 |
void G1DirtyCardQueueSet::handle_zero_index_for_thread(Thread* t) { |
49752
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49333
diff
changeset
|
106 |
G1ThreadLocalData::dirty_card_queue(t).handle_zero_index(); |
1374 | 107 |
} |
108 |
||
55498 | 109 |
void G1DirtyCardQueueSet::enqueue_completed_buffer(BufferNode* cbn) { |
57955
18863bf3501f
8230332: G1DirtyCardQueueSet _notify_when_complete is always true
kbarrett
parents:
57953
diff
changeset
|
110 |
MonitorLocker ml(_cbl_mon, Mutex::_no_safepoint_check_flag); |
55498 | 111 |
cbn->set_next(NULL); |
112 |
if (_completed_buffers_tail == NULL) { |
|
113 |
assert(_completed_buffers_head == NULL, "Well-formedness"); |
|
114 |
_completed_buffers_head = cbn; |
|
115 |
_completed_buffers_tail = cbn; |
|
116 |
} else { |
|
117 |
_completed_buffers_tail->set_next(cbn); |
|
118 |
_completed_buffers_tail = cbn; |
|
119 |
} |
|
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
120 |
_num_cards += buffer_size() - cbn->index(); |
55498 | 121 |
|
122 |
if (!process_completed_buffers() && |
|
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
123 |
(num_cards() > process_cards_threshold())) { |
55498 | 124 |
set_process_completed_buffers(true); |
57955
18863bf3501f
8230332: G1DirtyCardQueueSet _notify_when_complete is always true
kbarrett
parents:
57953
diff
changeset
|
125 |
ml.notify_all(); |
55498 | 126 |
} |
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
127 |
verify_num_cards(); |
55498 | 128 |
} |
129 |
||
130 |
BufferNode* G1DirtyCardQueueSet::get_completed_buffer(size_t stop_at) { |
|
131 |
MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag); |
|
132 |
||
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
133 |
if (num_cards() <= stop_at) { |
55498 | 134 |
return NULL; |
135 |
} |
|
136 |
||
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
137 |
assert(num_cards() > 0, "invariant"); |
55498 | 138 |
assert(_completed_buffers_head != NULL, "invariant"); |
139 |
assert(_completed_buffers_tail != NULL, "invariant"); |
|
140 |
||
141 |
BufferNode* bn = _completed_buffers_head; |
|
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
142 |
_num_cards -= buffer_size() - bn->index(); |
55498 | 143 |
_completed_buffers_head = bn->next(); |
144 |
if (_completed_buffers_head == NULL) { |
|
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
145 |
assert(num_cards() == 0, "invariant"); |
55498 | 146 |
_completed_buffers_tail = NULL; |
147 |
set_process_completed_buffers(false); |
|
148 |
} |
|
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
149 |
verify_num_cards(); |
55498 | 150 |
bn->set_next(NULL); |
151 |
return bn; |
|
152 |
} |
|
153 |
||
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
154 |
#ifdef ASSERT |
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
155 |
void G1DirtyCardQueueSet::verify_num_cards() const { |
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
156 |
size_t actual = 0; |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
157 |
BufferNode* cur = _completed_buffers_head; |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
158 |
while (cur != NULL) { |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
159 |
actual += buffer_size() - cur->index(); |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
160 |
cur = cur->next(); |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
161 |
} |
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
162 |
assert(actual == _num_cards, |
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
163 |
"Num entries in completed buffers should be " SIZE_FORMAT " but are " SIZE_FORMAT, |
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
164 |
_num_cards, actual); |
57507
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
165 |
} |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
166 |
#endif |
f6b30bd6804e
8227719: G1 Pending cards estimation too conservative in cost prediction
tschatzl
parents:
55752
diff
changeset
|
167 |
|
55498 | 168 |
void G1DirtyCardQueueSet::abandon_completed_buffers() { |
169 |
BufferNode* buffers_to_delete = NULL; |
|
170 |
{ |
|
171 |
MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag); |
|
172 |
buffers_to_delete = _completed_buffers_head; |
|
173 |
_completed_buffers_head = NULL; |
|
174 |
_completed_buffers_tail = NULL; |
|
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
175 |
_num_cards = 0; |
55498 | 176 |
set_process_completed_buffers(false); |
177 |
} |
|
178 |
while (buffers_to_delete != NULL) { |
|
179 |
BufferNode* bn = buffers_to_delete; |
|
180 |
buffers_to_delete = bn->next(); |
|
181 |
bn->set_next(NULL); |
|
182 |
deallocate_buffer(bn); |
|
183 |
} |
|
184 |
} |
|
185 |
||
186 |
void G1DirtyCardQueueSet::notify_if_necessary() { |
|
57955
18863bf3501f
8230332: G1DirtyCardQueueSet _notify_when_complete is always true
kbarrett
parents:
57953
diff
changeset
|
187 |
MonitorLocker ml(_cbl_mon, Mutex::_no_safepoint_check_flag); |
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
188 |
if (num_cards() > process_cards_threshold()) { |
55498 | 189 |
set_process_completed_buffers(true); |
57955
18863bf3501f
8230332: G1DirtyCardQueueSet _notify_when_complete is always true
kbarrett
parents:
57953
diff
changeset
|
190 |
ml.notify_all(); |
55498 | 191 |
} |
192 |
} |
|
193 |
||
194 |
// Merge lists of buffers. Notify the processing threads. |
|
195 |
// The source queue is emptied as a result. The queues |
|
196 |
// must share the monitor. |
|
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
197 |
void G1DirtyCardQueueSet::merge_bufferlists(G1RedirtyCardsQueueSet* src) { |
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
198 |
assert(allocator() == src->allocator(), "precondition"); |
57890
6bb824c45df1
8230192: Rename G1RedirtyCardsBufferList to G1BufferNodeList
kbarrett
parents:
57507
diff
changeset
|
199 |
const G1BufferNodeList from = src->take_all_completed_buffers(); |
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
200 |
if (from._head == NULL) return; |
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
201 |
|
55498 | 202 |
MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag); |
203 |
if (_completed_buffers_tail == NULL) { |
|
204 |
assert(_completed_buffers_head == NULL, "Well-formedness"); |
|
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
205 |
_completed_buffers_head = from._head; |
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
206 |
_completed_buffers_tail = from._tail; |
55498 | 207 |
} else { |
208 |
assert(_completed_buffers_head != NULL, "Well formedness"); |
|
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
209 |
_completed_buffers_tail->set_next(from._head); |
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
210 |
_completed_buffers_tail = from._tail; |
55498 | 211 |
} |
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
212 |
_num_cards += from._entry_count; |
55498 | 213 |
|
214 |
assert(_completed_buffers_head == NULL && _completed_buffers_tail == NULL || |
|
215 |
_completed_buffers_head != NULL && _completed_buffers_tail != NULL, |
|
216 |
"Sanity"); |
|
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
217 |
verify_num_cards(); |
55498 | 218 |
} |
219 |
||
58033
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
220 |
G1BufferNodeList G1DirtyCardQueueSet::take_all_completed_buffers() { |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
221 |
MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag); |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
222 |
G1BufferNodeList result(_completed_buffers_head, _completed_buffers_tail, _num_cards); |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
223 |
_completed_buffers_head = NULL; |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
224 |
_completed_buffers_tail = NULL; |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
225 |
_num_cards = 0; |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
226 |
return result; |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
227 |
} |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
228 |
|
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
229 |
bool G1DirtyCardQueueSet::refine_buffer(BufferNode* node, |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
230 |
uint worker_id, |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
231 |
size_t* total_refined_cards) { |
58033
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
232 |
G1RemSet* rem_set = G1CollectedHeap::heap()->rem_set(); |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
233 |
size_t size = buffer_size(); |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
234 |
void** buffer = BufferNode::make_buffer_from_node(node); |
46443 | 235 |
size_t i = node->index(); |
58033
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
236 |
assert(i <= size, "invariant"); |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
237 |
for ( ; (i < size) && !SuspendibleThreadSet::should_yield(); ++i) { |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
238 |
CardTable::CardValue* cp = static_cast<CardTable::CardValue*>(buffer[i]); |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
239 |
rem_set->refine_card_concurrently(cp, worker_id); |
37112
fe72d1d082ff
8151781: DirtyCardQueue::apply_closure is unused
kbarrett
parents:
37065
diff
changeset
|
240 |
} |
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
241 |
*total_refined_cards += (i - node->index()); |
55752
8ae33203d600
8162929: Enqueuing dirty cards into a single DCQS during GC does not scale
kbarrett
parents:
55498
diff
changeset
|
242 |
node->set_index(i); |
58033
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
243 |
return i == size; |
37112
fe72d1d082ff
8151781: DirtyCardQueue::apply_closure is unused
kbarrett
parents:
37065
diff
changeset
|
244 |
} |
fe72d1d082ff
8151781: DirtyCardQueue::apply_closure is unused
kbarrett
parents:
37065
diff
changeset
|
245 |
|
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
246 |
#ifndef ASSERT |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
247 |
#define assert_fully_consumed(node, buffer_size) |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
248 |
#else |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
249 |
#define assert_fully_consumed(node, buffer_size) \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
250 |
do { \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
251 |
size_t _afc_index = (node)->index(); \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
252 |
size_t _afc_size = (buffer_size); \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
253 |
assert(_afc_index == _afc_size, \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
254 |
"Buffer was not fully consumed as claimed: index: " \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
255 |
SIZE_FORMAT ", size: " SIZE_FORMAT, \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
256 |
_afc_index, _afc_size); \ |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
257 |
} while (0) |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
258 |
#endif // ASSERT |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
259 |
|
54970
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
260 |
bool G1DirtyCardQueueSet::process_or_enqueue_completed_buffer(BufferNode* node) { |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
261 |
if (Thread::current()->is_Java_thread()) { |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
262 |
// If the number of buffers exceeds the limit, make this Java |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
263 |
// thread do the processing itself. We don't lock to access |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
264 |
// buffer count or padding; it is fine to be imprecise here. The |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
265 |
// add of padding could overflow, which is treated as unlimited. |
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
266 |
size_t limit = max_cards() + max_cards_padding(); |
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
267 |
if ((num_cards() > limit) && (limit >= max_cards())) { |
54970
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
268 |
if (mut_process_buffer(node)) { |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
269 |
return true; |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
270 |
} |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
271 |
} |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
272 |
} |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
273 |
enqueue_completed_buffer(node); |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
274 |
return false; |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
275 |
} |
76d3d96a8bc2
8224167: Refactor PtrQueue completed buffer processing
kbarrett
parents:
54255
diff
changeset
|
276 |
|
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
277 |
bool G1DirtyCardQueueSet::mut_process_buffer(BufferNode* node) { |
57969
6f0215981777
8230327: Make G1DirtyCardQueueSet free-id init unconditional
kbarrett
parents:
57955
diff
changeset
|
278 |
uint worker_id = _free_ids.claim_par_id(); // temporarily claim an id |
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
279 |
uint counter_index = worker_id - par_ids_start(); |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
280 |
size_t* counter = &_mutator_refined_cards_counters[counter_index]; |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
281 |
bool result = refine_buffer(node, worker_id, counter); |
57969
6f0215981777
8230327: Make G1DirtyCardQueueSet free-id init unconditional
kbarrett
parents:
57955
diff
changeset
|
282 |
_free_ids.release_par_id(worker_id); // release the id |
1374 | 283 |
|
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
284 |
if (result) { |
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
285 |
assert_fully_consumed(node, buffer_size()); |
1374 | 286 |
} |
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
287 |
return result; |
1374 | 288 |
} |
289 |
||
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
290 |
bool G1DirtyCardQueueSet::refine_completed_buffer_concurrently(uint worker_id, |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
291 |
size_t stop_at, |
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
292 |
size_t* total_refined_cards) { |
58033
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
293 |
BufferNode* node = get_completed_buffer(stop_at); |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
294 |
if (node == NULL) { |
36179
03a0d7b8450f
8149793: DirtyCardQueueSet::apply_closure_to_completed_buffer_helper isn't helpful
kbarrett
parents:
35902
diff
changeset
|
295 |
return false; |
58508
d6058bd73982
8231153: Improve concurrent refinement statistics
kbarrett
parents:
58059
diff
changeset
|
296 |
} else if (refine_buffer(node, worker_id, total_refined_cards)) { |
58033
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
297 |
assert_fully_consumed(node, buffer_size()); |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
298 |
// Done with fully processed buffer. |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
299 |
deallocate_buffer(node); |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
300 |
return true; |
36179
03a0d7b8450f
8149793: DirtyCardQueueSet::apply_closure_to_completed_buffer_helper isn't helpful
kbarrett
parents:
35902
diff
changeset
|
301 |
} else { |
58033
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
302 |
// Return partially processed buffer to the queue. |
9162feb63c42
8230404: Refactor logged card refinement support in G1DirtyCardQueueSet
kbarrett
parents:
57969
diff
changeset
|
303 |
enqueue_completed_buffer(node); |
37197
282fa21230c3
8151670: Unexpected concurrent refinement deactivation and reactivation
kbarrett
parents:
37112
diff
changeset
|
304 |
return true; |
36179
03a0d7b8450f
8149793: DirtyCardQueueSet::apply_closure_to_completed_buffer_helper isn't helpful
kbarrett
parents:
35902
diff
changeset
|
305 |
} |
6247 | 306 |
} |
307 |
||
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
308 |
void G1DirtyCardQueueSet::abandon_logs() { |
6247 | 309 |
assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); |
53102
35530ca3e0b2
8214201: Make PtrQueueSet completed buffer list private
kbarrett
parents:
53034
diff
changeset
|
310 |
abandon_completed_buffers(); |
54006 | 311 |
|
1374 | 312 |
// Since abandon is done only at safepoints, we can safely manipulate |
313 |
// these queues. |
|
54006 | 314 |
struct AbandonThreadLogClosure : public ThreadClosure { |
315 |
virtual void do_thread(Thread* t) { |
|
316 |
G1ThreadLocalData::dirty_card_queue(t).reset(); |
|
317 |
} |
|
318 |
} closure; |
|
319 |
Threads::threads_do(&closure); |
|
320 |
||
54255 | 321 |
G1BarrierSet::shared_dirty_card_queue().reset(); |
37065 | 322 |
} |
1374 | 323 |
|
53747
13acc8e38a29
8218089: Rename DirtyCardQueue et al to follow usual G1 naming conventions
kbarrett
parents:
53482
diff
changeset
|
324 |
void G1DirtyCardQueueSet::concatenate_logs() { |
1374 | 325 |
// Iterate over all the threads, if we find a partial log add it to |
326 |
// the global list of logs. Temporarily turn off the limit on the number |
|
327 |
// of outstanding buffers. |
|
328 |
assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); |
|
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
329 |
size_t old_limit = max_cards(); |
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
330 |
set_max_cards(MaxCardsUnlimited); |
54006 | 331 |
|
54255 | 332 |
struct ConcatenateThreadLogClosure : public ThreadClosure { |
54006 | 333 |
virtual void do_thread(Thread* t) { |
54255 | 334 |
G1DirtyCardQueue& dcq = G1ThreadLocalData::dirty_card_queue(t); |
335 |
if (!dcq.is_empty()) { |
|
336 |
dcq.flush(); |
|
337 |
} |
|
54006 | 338 |
} |
54255 | 339 |
} closure; |
54006 | 340 |
Threads::threads_do(&closure); |
341 |
||
54255 | 342 |
G1BarrierSet::shared_dirty_card_queue().flush(); |
57953
d78c910f9069
8230109: G1DirtyCardQueueSet should use card counts rather than buffer counts
kbarrett
parents:
57890
diff
changeset
|
343 |
set_max_cards(old_limit); |
1374 | 344 |
} |