author | tschatzl |
Thu, 06 Aug 2015 15:52:33 +0200 | |
changeset 32187 | 0891f3fa84fc |
parent 32186 | 6f0257cb6493 |
child 32377 | 5ee15c417d02 |
permissions | -rw-r--r-- |
25482 | 1 |
/* |
29689
dc72789f83ae
8075249: Cleanup forward_to_atomic and ClaimedForwardPtr
stefank
parents:
28213
diff
changeset
|
2 |
* Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. |
25482 | 3 |
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
4 |
* |
|
5 |
* This code is free software; you can redistribute it and/or modify it |
|
6 |
* under the terms of the GNU General Public License version 2 only, as |
|
7 |
* published by the Free Software Foundation. |
|
8 |
* |
|
9 |
* This code is distributed in the hope that it will be useful, but WITHOUT |
|
10 |
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|
11 |
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
|
12 |
* version 2 for more details (a copy is included in the LICENSE file that |
|
13 |
* accompanied this code). |
|
14 |
* |
|
15 |
* You should have received a copy of the GNU General Public License version |
|
16 |
* 2 along with this work; if not, write to the Free Software Foundation, |
|
17 |
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
|
18 |
* |
|
19 |
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
20 |
* or visit www.oracle.com if you need additional information or have any |
|
21 |
* questions. |
|
22 |
* |
|
23 |
*/ |
|
24 |
||
25 |
#include "precompiled.hpp" |
|
30764 | 26 |
#include "gc/g1/g1CollectedHeap.inline.hpp" |
27 |
#include "gc/g1/g1OopClosures.inline.hpp" |
|
28 |
#include "gc/g1/g1ParScanThreadState.inline.hpp" |
|
29 |
#include "gc/g1/g1StringDedup.hpp" |
|
30 |
#include "gc/shared/taskqueue.inline.hpp" |
|
25482 | 31 |
#include "oops/oop.inline.hpp" |
32 |
#include "runtime/prefetch.inline.hpp" |
|
33 |
||
32187
0891f3fa84fc
8133047: Rename G1ParScanThreadState::_queue_num to _worker_id
tschatzl
parents:
32186
diff
changeset
|
34 |
G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h, uint worker_id, ReferenceProcessor* rp) |
25482 | 35 |
: _g1h(g1h), |
32187
0891f3fa84fc
8133047: Rename G1ParScanThreadState::_queue_num to _worker_id
tschatzl
parents:
32186
diff
changeset
|
36 |
_refs(g1h->task_queue(worker_id)), |
25482 | 37 |
_dcq(&g1h->dirty_card_queue_set()), |
38 |
_ct_bs(g1h->g1_barrier_set()), |
|
39 |
_g1_rem(g1h->g1_rem_set()), |
|
32187
0891f3fa84fc
8133047: Rename G1ParScanThreadState::_queue_num to _worker_id
tschatzl
parents:
32186
diff
changeset
|
40 |
_hash_seed(17), _worker_id(worker_id), |
25482 | 41 |
_term_attempts(0), |
28213
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
42 |
_tenuring_threshold(g1h->g1_policy()->tenuring_threshold()), |
25484
2cd3aff61672
8040977: G1 crashes when run with -XX:-G1DeferredRSUpdate
tschatzl
parents:
25483
diff
changeset
|
43 |
_age_table(false), _scanner(g1h, rp), |
26837
72a43d3841e7
8057536: Refactor G1 to allow context specific allocations
sjohanss
parents:
25905
diff
changeset
|
44 |
_strong_roots_time(0), _term_time(0) { |
25484
2cd3aff61672
8040977: G1 crashes when run with -XX:-G1DeferredRSUpdate
tschatzl
parents:
25483
diff
changeset
|
45 |
_scanner.set_par_scan_thread_state(this); |
25482 | 46 |
// we allocate G1YoungSurvRateNumRegions plus one entries, since |
47 |
// we "sacrifice" entry 0 to keep track of surviving bytes for |
|
48 |
// non-young regions (where the age is -1) |
|
49 |
// We also add a few elements at the beginning and at the end in |
|
50 |
// an attempt to eliminate cache contention |
|
51 |
uint real_length = 1 + _g1h->g1_policy()->young_cset_region_length(); |
|
52 |
uint array_length = PADDING_ELEM_NUM + |
|
53 |
real_length + |
|
54 |
PADDING_ELEM_NUM; |
|
55 |
_surviving_young_words_base = NEW_C_HEAP_ARRAY(size_t, array_length, mtGC); |
|
56 |
if (_surviving_young_words_base == NULL) |
|
57 |
vm_exit_out_of_memory(array_length * sizeof(size_t), OOM_MALLOC_ERROR, |
|
58 |
"Not enough space for young surv histo."); |
|
59 |
_surviving_young_words = _surviving_young_words_base + PADDING_ELEM_NUM; |
|
60 |
memset(_surviving_young_words, 0, (size_t) real_length * sizeof(size_t)); |
|
61 |
||
32185
49a57ff2c3cb
8073052: Rename and clean up the allocation manager hierarchy in g1Allocator.?pp
tschatzl
parents:
31976
diff
changeset
|
62 |
_plab_allocator = G1PLABAllocator::create_allocator(_g1h->allocator()); |
25482 | 63 |
|
28213
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
64 |
_dest[InCSetState::NotInCSet] = InCSetState::NotInCSet; |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
65 |
// The dest for Young is used when the objects are aged enough to |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
66 |
// need to be moved to the next space. |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
67 |
_dest[InCSetState::Young] = InCSetState::Old; |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
68 |
_dest[InCSetState::Old] = InCSetState::Old; |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
69 |
|
25482 | 70 |
_start = os::elapsedTime(); |
71 |
} |
|
72 |
||
25483
962ccf95c515
8035401: Fix visibility of G1ParScanThreadState members
tschatzl
parents:
25482
diff
changeset
|
73 |
G1ParScanThreadState::~G1ParScanThreadState() { |
32185
49a57ff2c3cb
8073052: Rename and clean up the allocation manager hierarchy in g1Allocator.?pp
tschatzl
parents:
31976
diff
changeset
|
74 |
_plab_allocator->retire_alloc_buffers(); |
49a57ff2c3cb
8073052: Rename and clean up the allocation manager hierarchy in g1Allocator.?pp
tschatzl
parents:
31976
diff
changeset
|
75 |
delete _plab_allocator; |
27880
afb974a04396
8060074: os::free() takes MemoryTrackingLevel but doesn't need it
coleenp
parents:
26837
diff
changeset
|
76 |
FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_base); |
25483
962ccf95c515
8035401: Fix visibility of G1ParScanThreadState members
tschatzl
parents:
25482
diff
changeset
|
77 |
} |
962ccf95c515
8035401: Fix visibility of G1ParScanThreadState members
tschatzl
parents:
25482
diff
changeset
|
78 |
|
32186
6f0257cb6493
8133043: Clean up code related to termination stats printing
tschatzl
parents:
32185
diff
changeset
|
79 |
void G1ParScanThreadState::print_termination_stats_hdr(outputStream* const st) { |
25482 | 80 |
st->print_raw_cr("GC Termination Stats"); |
32186
6f0257cb6493
8133043: Clean up code related to termination stats printing
tschatzl
parents:
32185
diff
changeset
|
81 |
st->print_raw_cr(" elapsed --strong roots-- -------termination------- ------waste (KiB)------"); |
6f0257cb6493
8133043: Clean up code related to termination stats printing
tschatzl
parents:
32185
diff
changeset
|
82 |
st->print_raw_cr("thr ms ms % ms % attempts total alloc undo"); |
6f0257cb6493
8133043: Clean up code related to termination stats printing
tschatzl
parents:
32185
diff
changeset
|
83 |
st->print_raw_cr("--- --------- --------- ------ --------- ------ -------- ------- ------- -------"); |
25482 | 84 |
} |
85 |
||
32186
6f0257cb6493
8133043: Clean up code related to termination stats printing
tschatzl
parents:
32185
diff
changeset
|
86 |
void G1ParScanThreadState::print_termination_stats(outputStream* const st) const { |
25482 | 87 |
const double elapsed_ms = elapsed_time() * 1000.0; |
88 |
const double s_roots_ms = strong_roots_time() * 1000.0; |
|
89 |
const double term_ms = term_time() * 1000.0; |
|
30564
a37d98a1eb54
8071462: Remove G1ParGCAllocator::alloc_buffer_waste
tschatzl
parents:
30175
diff
changeset
|
90 |
size_t alloc_buffer_waste = 0; |
a37d98a1eb54
8071462: Remove G1ParGCAllocator::alloc_buffer_waste
tschatzl
parents:
30175
diff
changeset
|
91 |
size_t undo_waste = 0; |
32185
49a57ff2c3cb
8073052: Rename and clean up the allocation manager hierarchy in g1Allocator.?pp
tschatzl
parents:
31976
diff
changeset
|
92 |
_plab_allocator->waste(alloc_buffer_waste, undo_waste); |
32186
6f0257cb6493
8133043: Clean up code related to termination stats printing
tschatzl
parents:
32185
diff
changeset
|
93 |
st->print_cr("%3u %9.2f %9.2f %6.2f " |
25482 | 94 |
"%9.2f %6.2f " SIZE_FORMAT_W(8) " " |
95 |
SIZE_FORMAT_W(7) " " SIZE_FORMAT_W(7) " " SIZE_FORMAT_W(7), |
|
32187
0891f3fa84fc
8133047: Rename G1ParScanThreadState::_queue_num to _worker_id
tschatzl
parents:
32186
diff
changeset
|
96 |
_worker_id, elapsed_ms, s_roots_ms, s_roots_ms * 100 / elapsed_ms, |
25482 | 97 |
term_ms, term_ms * 100 / elapsed_ms, term_attempts(), |
26837
72a43d3841e7
8057536: Refactor G1 to allow context specific allocations
sjohanss
parents:
25905
diff
changeset
|
98 |
(alloc_buffer_waste + undo_waste) * HeapWordSize / K, |
72a43d3841e7
8057536: Refactor G1 to allow context specific allocations
sjohanss
parents:
25905
diff
changeset
|
99 |
alloc_buffer_waste * HeapWordSize / K, |
72a43d3841e7
8057536: Refactor G1 to allow context specific allocations
sjohanss
parents:
25905
diff
changeset
|
100 |
undo_waste * HeapWordSize / K); |
25482 | 101 |
} |
102 |
||
103 |
#ifdef ASSERT |
|
104 |
bool G1ParScanThreadState::verify_ref(narrowOop* ref) const { |
|
105 |
assert(ref != NULL, "invariant"); |
|
106 |
assert(UseCompressedOops, "sanity"); |
|
107 |
assert(!has_partial_array_mask(ref), err_msg("ref=" PTR_FORMAT, p2i(ref))); |
|
108 |
oop p = oopDesc::load_decode_heap_oop(ref); |
|
109 |
assert(_g1h->is_in_g1_reserved(p), |
|
110 |
err_msg("ref=" PTR_FORMAT " p=" PTR_FORMAT, p2i(ref), p2i(p))); |
|
111 |
return true; |
|
112 |
} |
|
113 |
||
114 |
bool G1ParScanThreadState::verify_ref(oop* ref) const { |
|
115 |
assert(ref != NULL, "invariant"); |
|
116 |
if (has_partial_array_mask(ref)) { |
|
117 |
// Must be in the collection set--it's already been copied. |
|
118 |
oop p = clear_partial_array_mask(ref); |
|
119 |
assert(_g1h->obj_in_cs(p), |
|
120 |
err_msg("ref=" PTR_FORMAT " p=" PTR_FORMAT, p2i(ref), p2i(p))); |
|
121 |
} else { |
|
122 |
oop p = oopDesc::load_decode_heap_oop(ref); |
|
123 |
assert(_g1h->is_in_g1_reserved(p), |
|
124 |
err_msg("ref=" PTR_FORMAT " p=" PTR_FORMAT, p2i(ref), p2i(p))); |
|
125 |
} |
|
126 |
return true; |
|
127 |
} |
|
128 |
||
129 |
bool G1ParScanThreadState::verify_task(StarTask ref) const { |
|
130 |
if (ref.is_narrow()) { |
|
131 |
return verify_ref((narrowOop*) ref); |
|
132 |
} else { |
|
133 |
return verify_ref((oop*) ref); |
|
134 |
} |
|
135 |
} |
|
136 |
#endif // ASSERT |
|
137 |
||
138 |
void G1ParScanThreadState::trim_queue() { |
|
139 |
StarTask ref; |
|
140 |
do { |
|
141 |
// Drain the overflow stack first, so other threads can steal. |
|
25483
962ccf95c515
8035401: Fix visibility of G1ParScanThreadState members
tschatzl
parents:
25482
diff
changeset
|
142 |
while (_refs->pop_overflow(ref)) { |
962ccf95c515
8035401: Fix visibility of G1ParScanThreadState members
tschatzl
parents:
25482
diff
changeset
|
143 |
dispatch_reference(ref); |
25482 | 144 |
} |
145 |
||
25483
962ccf95c515
8035401: Fix visibility of G1ParScanThreadState members
tschatzl
parents:
25482
diff
changeset
|
146 |
while (_refs->pop_local(ref)) { |
962ccf95c515
8035401: Fix visibility of G1ParScanThreadState members
tschatzl
parents:
25482
diff
changeset
|
147 |
dispatch_reference(ref); |
25482 | 148 |
} |
25483
962ccf95c515
8035401: Fix visibility of G1ParScanThreadState members
tschatzl
parents:
25482
diff
changeset
|
149 |
} while (!_refs->is_empty()); |
25482 | 150 |
} |
151 |
||
28213
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
152 |
HeapWord* G1ParScanThreadState::allocate_in_next_plab(InCSetState const state, |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
153 |
InCSetState* dest, |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
154 |
size_t word_sz, |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
155 |
AllocationContext_t const context) { |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
156 |
assert(state.is_in_cset_or_humongous(), err_msg("Unexpected state: " CSETSTATE_FORMAT, state.value())); |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
157 |
assert(dest->is_in_cset_or_humongous(), err_msg("Unexpected dest: " CSETSTATE_FORMAT, dest->value())); |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
158 |
|
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
159 |
// Right now we only have two types of regions (young / old) so |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
160 |
// let's keep the logic here simple. We can generalize it when necessary. |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
161 |
if (dest->is_young()) { |
32185
49a57ff2c3cb
8073052: Rename and clean up the allocation manager hierarchy in g1Allocator.?pp
tschatzl
parents:
31976
diff
changeset
|
162 |
HeapWord* const obj_ptr = _plab_allocator->allocate(InCSetState::Old, |
49a57ff2c3cb
8073052: Rename and clean up the allocation manager hierarchy in g1Allocator.?pp
tschatzl
parents:
31976
diff
changeset
|
163 |
word_sz, |
49a57ff2c3cb
8073052: Rename and clean up the allocation manager hierarchy in g1Allocator.?pp
tschatzl
parents:
31976
diff
changeset
|
164 |
context); |
28213
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
165 |
if (obj_ptr == NULL) { |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
166 |
return NULL; |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
167 |
} |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
168 |
// Make sure that we won't attempt to copy any other objects out |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
169 |
// of a survivor region (given that apparently we cannot allocate |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
170 |
// any new ones) to avoid coming into this slow path. |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
171 |
_tenuring_threshold = 0; |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
172 |
dest->set_old(); |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
173 |
return obj_ptr; |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
174 |
} else { |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
175 |
assert(dest->is_old(), err_msg("Unexpected dest: " CSETSTATE_FORMAT, dest->value())); |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
176 |
// no other space to try. |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
177 |
return NULL; |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
178 |
} |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
179 |
} |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
180 |
|
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
181 |
InCSetState G1ParScanThreadState::next_state(InCSetState const state, markOop const m, uint& age) { |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
182 |
if (state.is_young()) { |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
183 |
age = !m->has_displaced_mark_helper() ? m->age() |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
184 |
: m->displaced_mark_helper()->age(); |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
185 |
if (age < _tenuring_threshold) { |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
186 |
return state; |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
187 |
} |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
188 |
} |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
189 |
return dest(state); |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
190 |
} |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
191 |
|
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
192 |
oop G1ParScanThreadState::copy_to_survivor_space(InCSetState const state, |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
193 |
oop const old, |
27682
dbd1c3f92130
8064473: Improved handling of age during object copy in G1
sfriberg
parents:
26837
diff
changeset
|
194 |
markOop const old_mark) { |
28213
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
195 |
const size_t word_sz = old->size(); |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
196 |
HeapRegion* const from_region = _g1h->heap_region_containing_raw(old); |
25482 | 197 |
// +1 to make the -1 indexes valid... |
28213
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
198 |
const int young_index = from_region->young_index_in_cset()+1; |
25482 | 199 |
assert( (from_region->is_young() && young_index > 0) || |
200 |
(!from_region->is_young() && young_index == 0), "invariant" ); |
|
28213
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
201 |
const AllocationContext_t context = from_region->allocation_context(); |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
202 |
|
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
203 |
uint age = 0; |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
204 |
InCSetState dest_state = next_state(state, old_mark, age); |
32185
49a57ff2c3cb
8073052: Rename and clean up the allocation manager hierarchy in g1Allocator.?pp
tschatzl
parents:
31976
diff
changeset
|
205 |
HeapWord* obj_ptr = _plab_allocator->plab_allocate(dest_state, word_sz, context); |
28213
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
206 |
|
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
207 |
// PLAB allocations should succeed most of the time, so we'll |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
208 |
// normally check against NULL once and that's it. |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
209 |
if (obj_ptr == NULL) { |
32185
49a57ff2c3cb
8073052: Rename and clean up the allocation manager hierarchy in g1Allocator.?pp
tschatzl
parents:
31976
diff
changeset
|
210 |
obj_ptr = _plab_allocator->allocate_direct_or_new_plab(dest_state, word_sz, context); |
28213
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
211 |
if (obj_ptr == NULL) { |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
212 |
obj_ptr = allocate_in_next_plab(state, &dest_state, word_sz, context); |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
213 |
if (obj_ptr == NULL) { |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
214 |
// This will either forward-to-self, or detect that someone else has |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
215 |
// installed a forwarding pointer. |
31976
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
216 |
return handle_evacuation_failure_par(old, old_mark); |
28213
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
217 |
} |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
218 |
} |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
219 |
} |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
220 |
|
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
221 |
assert(obj_ptr != NULL, "when we get here, allocation should have succeeded"); |
29689
dc72789f83ae
8075249: Cleanup forward_to_atomic and ClaimedForwardPtr
stefank
parents:
28213
diff
changeset
|
222 |
assert(_g1h->is_in_reserved(obj_ptr), "Allocated memory should be in the heap"); |
dc72789f83ae
8075249: Cleanup forward_to_atomic and ClaimedForwardPtr
stefank
parents:
28213
diff
changeset
|
223 |
|
25482 | 224 |
#ifndef PRODUCT |
225 |
// Should this evacuation fail? |
|
226 |
if (_g1h->evacuation_should_fail()) { |
|
28213
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
227 |
// Doing this after all the allocation attempts also tests the |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
228 |
// undo_allocation() method too. |
32185
49a57ff2c3cb
8073052: Rename and clean up the allocation manager hierarchy in g1Allocator.?pp
tschatzl
parents:
31976
diff
changeset
|
229 |
_plab_allocator->undo_allocation(dest_state, obj_ptr, word_sz, context); |
31976
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
230 |
return handle_evacuation_failure_par(old, old_mark); |
25482 | 231 |
} |
232 |
#endif // !PRODUCT |
|
233 |
||
234 |
// We're going to allocate linearly, so might as well prefetch ahead. |
|
235 |
Prefetch::write(obj_ptr, PrefetchCopyIntervalInBytes); |
|
236 |
||
28213
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
237 |
const oop obj = oop(obj_ptr); |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
238 |
const oop forward_ptr = old->forward_to_atomic(obj); |
25482 | 239 |
if (forward_ptr == NULL) { |
240 |
Copy::aligned_disjoint_words((HeapWord*) old, obj_ptr, word_sz); |
|
241 |
||
28213
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
242 |
if (dest_state.is_young()) { |
27682
dbd1c3f92130
8064473: Improved handling of age during object copy in G1
sfriberg
parents:
26837
diff
changeset
|
243 |
if (age < markOopDesc::max_age) { |
dbd1c3f92130
8064473: Improved handling of age during object copy in G1
sfriberg
parents:
26837
diff
changeset
|
244 |
age++; |
dbd1c3f92130
8064473: Improved handling of age during object copy in G1
sfriberg
parents:
26837
diff
changeset
|
245 |
} |
dbd1c3f92130
8064473: Improved handling of age during object copy in G1
sfriberg
parents:
26837
diff
changeset
|
246 |
if (old_mark->has_displaced_mark_helper()) { |
dbd1c3f92130
8064473: Improved handling of age during object copy in G1
sfriberg
parents:
26837
diff
changeset
|
247 |
// In this case, we have to install the mark word first, |
25482 | 248 |
// otherwise obj looks to be forwarded (the old mark word, |
249 |
// which contains the forward pointer, was copied) |
|
27682
dbd1c3f92130
8064473: Improved handling of age during object copy in G1
sfriberg
parents:
26837
diff
changeset
|
250 |
obj->set_mark(old_mark); |
dbd1c3f92130
8064473: Improved handling of age during object copy in G1
sfriberg
parents:
26837
diff
changeset
|
251 |
markOop new_mark = old_mark->displaced_mark_helper()->set_age(age); |
dbd1c3f92130
8064473: Improved handling of age during object copy in G1
sfriberg
parents:
26837
diff
changeset
|
252 |
old_mark->set_displaced_mark_helper(new_mark); |
25482 | 253 |
} else { |
27682
dbd1c3f92130
8064473: Improved handling of age during object copy in G1
sfriberg
parents:
26837
diff
changeset
|
254 |
obj->set_mark(old_mark->set_age(age)); |
25482 | 255 |
} |
27682
dbd1c3f92130
8064473: Improved handling of age during object copy in G1
sfriberg
parents:
26837
diff
changeset
|
256 |
age_table()->add(age, word_sz); |
25482 | 257 |
} else { |
27682
dbd1c3f92130
8064473: Improved handling of age during object copy in G1
sfriberg
parents:
26837
diff
changeset
|
258 |
obj->set_mark(old_mark); |
25482 | 259 |
} |
260 |
||
261 |
if (G1StringDedup::is_enabled()) { |
|
28213
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
262 |
const bool is_from_young = state.is_young(); |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
263 |
const bool is_to_young = dest_state.is_young(); |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
264 |
assert(is_from_young == _g1h->heap_region_containing_raw(old)->is_young(), |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
265 |
"sanity"); |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
266 |
assert(is_to_young == _g1h->heap_region_containing_raw(obj)->is_young(), |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
267 |
"sanity"); |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
268 |
G1StringDedup::enqueue_from_evacuation(is_from_young, |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
269 |
is_to_young, |
32187
0891f3fa84fc
8133047: Rename G1ParScanThreadState::_queue_num to _worker_id
tschatzl
parents:
32186
diff
changeset
|
270 |
_worker_id, |
25482 | 271 |
obj); |
272 |
} |
|
273 |
||
28213
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
274 |
size_t* const surv_young_words = surviving_young_words(); |
25482 | 275 |
surv_young_words[young_index] += word_sz; |
276 |
||
277 |
if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) { |
|
278 |
// We keep track of the next start index in the length field of |
|
279 |
// the to-space object. The actual length can be found in the |
|
280 |
// length field of the from-space object. |
|
281 |
arrayOop(obj)->set_length(0); |
|
282 |
oop* old_p = set_partial_array_mask(old); |
|
283 |
push_on_queue(old_p); |
|
284 |
} else { |
|
28213
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
285 |
HeapRegion* const to_region = _g1h->heap_region_containing_raw(obj_ptr); |
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
286 |
_scanner.set_region(to_region); |
25482 | 287 |
obj->oop_iterate_backwards(&_scanner); |
288 |
} |
|
28213
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
289 |
return obj; |
25482 | 290 |
} else { |
32185
49a57ff2c3cb
8073052: Rename and clean up the allocation manager hierarchy in g1Allocator.?pp
tschatzl
parents:
31976
diff
changeset
|
291 |
_plab_allocator->undo_allocation(dest_state, obj_ptr, word_sz, context); |
28213
b0bf57cd1e9d
8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents:
27883
diff
changeset
|
292 |
return forward_ptr; |
25482 | 293 |
} |
294 |
} |
|
31976
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
295 |
|
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
296 |
oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markOop m) { |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
297 |
assert(_g1h->obj_in_cs(old), |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
298 |
err_msg("Object " PTR_FORMAT " should be in the CSet", p2i(old))); |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
299 |
|
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
300 |
oop forward_ptr = old->forward_to_atomic(old); |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
301 |
if (forward_ptr == NULL) { |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
302 |
// Forward-to-self succeeded. We are the "owner" of the object. |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
303 |
HeapRegion* r = _g1h->heap_region_containing(old); |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
304 |
|
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
305 |
if (!r->evacuation_failed()) { |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
306 |
r->set_evacuation_failed(true); |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
307 |
_g1h->hr_printer()->evac_failure(r); |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
308 |
} |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
309 |
|
32187
0891f3fa84fc
8133047: Rename G1ParScanThreadState::_queue_num to _worker_id
tschatzl
parents:
32186
diff
changeset
|
310 |
_g1h->preserve_mark_during_evac_failure(_worker_id, old, m); |
31976
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
311 |
|
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
312 |
_scanner.set_region(r); |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
313 |
old->oop_iterate_backwards(&_scanner); |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
314 |
|
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
315 |
return old; |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
316 |
} else { |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
317 |
// Forward-to-self failed. Either someone else managed to allocate |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
318 |
// space for this object (old != forward_ptr) or they beat us in |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
319 |
// self-forwarding it (old == forward_ptr). |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
320 |
assert(old == forward_ptr || !_g1h->obj_in_cs(forward_ptr), |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
321 |
err_msg("Object " PTR_FORMAT " forwarded to: " PTR_FORMAT " " |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
322 |
"should not be in the CSet", |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
323 |
p2i(old), p2i(forward_ptr))); |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
324 |
return forward_ptr; |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
325 |
} |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
326 |
} |
da45f85bf4e1
8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents:
30764
diff
changeset
|
327 |