author | stefank |
Tue, 26 Nov 2019 10:47:46 +0100 | |
changeset 59290 | 97d13893ec3c |
parent 58059 | baa4dd528de0 |
permissions | -rw-r--r-- |
1374 | 1 |
/* |
53404 | 2 |
* Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved. |
1374 | 3 |
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
4 |
* |
|
5 |
* This code is free software; you can redistribute it and/or modify it |
|
6 |
* under the terms of the GNU General Public License version 2 only, as |
|
7 |
* published by the Free Software Foundation. |
|
8 |
* |
|
9 |
* This code is distributed in the hope that it will be useful, but WITHOUT |
|
10 |
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|
11 |
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
|
12 |
* version 2 for more details (a copy is included in the LICENSE file that |
|
13 |
* accompanied this code). |
|
14 |
* |
|
15 |
* You should have received a copy of the GNU General Public License version |
|
16 |
* 2 along with this work; if not, write to the Free Software Foundation, |
|
17 |
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
|
18 |
* |
|
5547
f4b087cbb361
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
5082
diff
changeset
|
19 |
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
f4b087cbb361
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
5082
diff
changeset
|
20 |
* or visit www.oracle.com if you need additional information or have any |
f4b087cbb361
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
5082
diff
changeset
|
21 |
* questions. |
1374 | 22 |
* |
23 |
*/ |
|
24 |
||
7397 | 25 |
#include "precompiled.hpp" |
49347
edb65305d3ac
8195148: Collapse G1SATBCardTableModRefBS and G1SATBCardTableLoggingModRefBS into a single G1BarrierSet
eosterlund
parents:
49164
diff
changeset
|
26 |
#include "gc/g1/g1BarrierSet.inline.hpp" |
49484
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
27 |
#include "gc/g1/g1BarrierSetAssembler.hpp" |
49164
7e958a8ebcd3
8195142: Refactor out card table from CardTableModRefBS to flatten the BarrierSet hierarchy
eosterlund
parents:
49007
diff
changeset
|
28 |
#include "gc/g1/g1CardTable.inline.hpp" |
30764 | 29 |
#include "gc/g1/g1CollectedHeap.inline.hpp" |
51441
2e91d927e00c
8154343: Make SATB related code available to other GCs
kbarrett
parents:
51390
diff
changeset
|
30 |
#include "gc/g1/g1SATBMarkQueueSet.hpp" |
49752
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49751
diff
changeset
|
31 |
#include "gc/g1/g1ThreadLocalData.hpp" |
30764 | 32 |
#include "gc/g1/heapRegion.hpp" |
51441
2e91d927e00c
8154343: Make SATB related code available to other GCs
kbarrett
parents:
51390
diff
changeset
|
33 |
#include "gc/shared/satbMarkQueue.hpp" |
35061 | 34 |
#include "logging/log.hpp" |
49592
77fb0be7d19f
8199946: Move load/store and encode/decode out of oopDesc
stefank
parents:
49484
diff
changeset
|
35 |
#include "oops/access.inline.hpp" |
77fb0be7d19f
8199946: Move load/store and encode/decode out of oopDesc
stefank
parents:
49484
diff
changeset
|
36 |
#include "oops/compressedOops.inline.hpp" |
29081
c61eb4914428
8072911: Remove includes of oop.inline.hpp from .hpp files
stefank
parents:
28830
diff
changeset
|
37 |
#include "oops/oop.inline.hpp" |
50180 | 38 |
#include "runtime/interfaceSupport.inline.hpp" |
7397 | 39 |
#include "runtime/mutexLocker.hpp" |
59290
97d13893ec3c
8234748: Clean up atomic and orderAccess includes
stefank
parents:
58059
diff
changeset
|
40 |
#include "runtime/orderAccess.hpp" |
14583
d70ee55535f4
8003935: Simplify the needed includes for using Thread::current()
stefank
parents:
10565
diff
changeset
|
41 |
#include "runtime/thread.inline.hpp" |
49484
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
42 |
#include "utilities/macros.hpp" |
49906 | 43 |
#ifdef COMPILER1 |
44 |
#include "gc/g1/c1/g1BarrierSetC1.hpp" |
|
45 |
#endif |
|
50180 | 46 |
#ifdef COMPILER2 |
47 |
#include "gc/g1/c2/g1BarrierSetC2.hpp" |
|
48 |
#endif |
|
49906 | 49 |
|
50 |
class G1BarrierSetC1; |
|
50180 | 51 |
class G1BarrierSetC2; |
1374 | 52 |
|
49347
edb65305d3ac
8195148: Collapse G1SATBCardTableModRefBS and G1SATBCardTableLoggingModRefBS into a single G1BarrierSet
eosterlund
parents:
49164
diff
changeset
|
53 |
G1BarrierSet::G1BarrierSet(G1CardTable* card_table) : |
49484
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
54 |
CardTableBarrierSet(make_barrier_set_assembler<G1BarrierSetAssembler>(), |
49906 | 55 |
make_barrier_set_c1<G1BarrierSetC1>(), |
50180 | 56 |
make_barrier_set_c2<G1BarrierSetC2>(), |
49484
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
57 |
card_table, |
51390
135285dfc551
8209396: Make PtrQueueSets not statically allocated
kbarrett
parents:
51368
diff
changeset
|
58 |
BarrierSet::FakeRtti(BarrierSet::G1BarrierSet)), |
53404 | 59 |
_satb_mark_queue_buffer_allocator("SATB Buffer Allocator", G1SATBBufferSize), |
60 |
_dirty_card_queue_buffer_allocator("DC Buffer Allocator", G1UpdateBufferSize), |
|
58059
baa4dd528de0
8221361: Eliminate two-phase initialization for PtrQueueSet classes
kbarrett
parents:
54366
diff
changeset
|
61 |
_satb_mark_queue_set(&_satb_mark_queue_buffer_allocator), |
baa4dd528de0
8221361: Eliminate two-phase initialization for PtrQueueSet classes
kbarrett
parents:
54366
diff
changeset
|
62 |
_dirty_card_queue_set(DirtyCardQ_CBL_mon, &_dirty_card_queue_buffer_allocator), |
54255 | 63 |
_shared_dirty_card_queue(&_dirty_card_queue_set) |
51390
135285dfc551
8209396: Make PtrQueueSets not statically allocated
kbarrett
parents:
51368
diff
changeset
|
64 |
{} |
1374 | 65 |
|
49347
edb65305d3ac
8195148: Collapse G1SATBCardTableModRefBS and G1SATBCardTableLoggingModRefBS into a single G1BarrierSet
eosterlund
parents:
49164
diff
changeset
|
66 |
void G1BarrierSet::enqueue(oop pre_val) { |
9176
42d9d1010f38
7009266: G1: assert(obj->is_oop_or_null(true )) failed: Error
johnc
parents:
7397
diff
changeset
|
67 |
// Nulls should have been already filtered. |
46968
9119841280f4
8160399: is_oop_or_null involves undefined behavior
coleenp
parents:
42598
diff
changeset
|
68 |
assert(oopDesc::is_oop(pre_val, true), "Error"); |
54006 | 69 |
G1ThreadLocalData::satb_mark_queue(Thread::current()).enqueue(pre_val); |
1374 | 70 |
} |
71 |
||
3262
30d1c247fc25
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
1374
diff
changeset
|
72 |
template <class T> void |
49484
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
73 |
G1BarrierSet::write_ref_array_pre_work(T* dst, size_t count) { |
49751
c3a10df652c0
8201316: Move G1-related static members from JavaThread to G1BarrierSet
pliden
parents:
49679
diff
changeset
|
74 |
if (!_satb_mark_queue_set.is_active()) return; |
3262
30d1c247fc25
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
1374
diff
changeset
|
75 |
T* elem_ptr = dst; |
49484
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
76 |
for (size_t i = 0; i < count; i++, elem_ptr++) { |
49592
77fb0be7d19f
8199946: Move load/store and encode/decode out of oopDesc
stefank
parents:
49484
diff
changeset
|
77 |
T heap_oop = RawAccess<>::oop_load(elem_ptr); |
77fb0be7d19f
8199946: Move load/store and encode/decode out of oopDesc
stefank
parents:
49484
diff
changeset
|
78 |
if (!CompressedOops::is_null(heap_oop)) { |
77fb0be7d19f
8199946: Move load/store and encode/decode out of oopDesc
stefank
parents:
49484
diff
changeset
|
79 |
enqueue(CompressedOops::decode_not_null(heap_oop)); |
3262
30d1c247fc25
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
1374
diff
changeset
|
80 |
} |
1374 | 81 |
} |
82 |
} |
|
83 |
||
49484
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
84 |
void G1BarrierSet::write_ref_array_pre(oop* dst, size_t count, bool dest_uninitialized) { |
25480
190facf4e732
8048214: Linker error when compiling G1SATBCardTableModRefBS after include order changes
mgerdin
parents:
25351
diff
changeset
|
85 |
if (!dest_uninitialized) { |
190facf4e732
8048214: Linker error when compiling G1SATBCardTableModRefBS after include order changes
mgerdin
parents:
25351
diff
changeset
|
86 |
write_ref_array_pre_work(dst, count); |
190facf4e732
8048214: Linker error when compiling G1SATBCardTableModRefBS after include order changes
mgerdin
parents:
25351
diff
changeset
|
87 |
} |
190facf4e732
8048214: Linker error when compiling G1SATBCardTableModRefBS after include order changes
mgerdin
parents:
25351
diff
changeset
|
88 |
} |
47998
fb0275c320a0
8189871: Refactor GC barriers to use declarative semantics
eosterlund
parents:
47794
diff
changeset
|
89 |
|
49484
ee8fa73b90f9
8198949: Modularize arraycopy stub routine GC barriers
eosterlund
parents:
49455
diff
changeset
|
90 |
void G1BarrierSet::write_ref_array_pre(narrowOop* dst, size_t count, bool dest_uninitialized) { |
25480
190facf4e732
8048214: Linker error when compiling G1SATBCardTableModRefBS after include order changes
mgerdin
parents:
25351
diff
changeset
|
91 |
if (!dest_uninitialized) { |
190facf4e732
8048214: Linker error when compiling G1SATBCardTableModRefBS after include order changes
mgerdin
parents:
25351
diff
changeset
|
92 |
write_ref_array_pre_work(dst, count); |
190facf4e732
8048214: Linker error when compiling G1SATBCardTableModRefBS after include order changes
mgerdin
parents:
25351
diff
changeset
|
93 |
} |
190facf4e732
8048214: Linker error when compiling G1SATBCardTableModRefBS after include order changes
mgerdin
parents:
25351
diff
changeset
|
94 |
} |
190facf4e732
8048214: Linker error when compiling G1SATBCardTableModRefBS after include order changes
mgerdin
parents:
25351
diff
changeset
|
95 |
|
54110 | 96 |
void G1BarrierSet::write_ref_field_post_slow(volatile CardValue* byte) { |
47998
fb0275c320a0
8189871: Refactor GC barriers to use declarative semantics
eosterlund
parents:
47794
diff
changeset
|
97 |
// In the slow path, we know a card is not young |
49164
7e958a8ebcd3
8195142: Refactor out card table from CardTableModRefBS to flatten the BarrierSet hierarchy
eosterlund
parents:
49007
diff
changeset
|
98 |
assert(*byte != G1CardTable::g1_young_card_val(), "slow path invoked without filtering"); |
20403
45a89fbcd8f7
8014555: G1: Memory ordering problem with Conc refinement and card marking
mgerdin
parents:
20309
diff
changeset
|
99 |
OrderAccess::storeload(); |
49164
7e958a8ebcd3
8195142: Refactor out card table from CardTableModRefBS to flatten the BarrierSet hierarchy
eosterlund
parents:
49007
diff
changeset
|
100 |
if (*byte != G1CardTable::dirty_card_val()) { |
7e958a8ebcd3
8195142: Refactor out card table from CardTableModRefBS to flatten the BarrierSet hierarchy
eosterlund
parents:
49007
diff
changeset
|
101 |
*byte = G1CardTable::dirty_card_val(); |
1374 | 102 |
Thread* thr = Thread::current(); |
54006 | 103 |
G1ThreadLocalData::dirty_card_queue(thr).enqueue(byte); |
1374 | 104 |
} |
105 |
} |
|
106 |
||
49347
edb65305d3ac
8195148: Collapse G1SATBCardTableModRefBS and G1SATBCardTableLoggingModRefBS into a single G1BarrierSet
eosterlund
parents:
49164
diff
changeset
|
107 |
void G1BarrierSet::invalidate(MemRegion mr) { |
48102
08be4c1e540e
8182050: assert(_whole_heap.contains(p)) failed: Attempt to access p out of bounds of card marking array's _whole_heap
tschatzl
parents:
47998
diff
changeset
|
108 |
if (mr.is_empty()) { |
08be4c1e540e
8182050: assert(_whole_heap.contains(p)) failed: Attempt to access p out of bounds of card marking array's _whole_heap
tschatzl
parents:
47998
diff
changeset
|
109 |
return; |
08be4c1e540e
8182050: assert(_whole_heap.contains(p)) failed: Attempt to access p out of bounds of card marking array's _whole_heap
tschatzl
parents:
47998
diff
changeset
|
110 |
} |
54110 | 111 |
volatile CardValue* byte = _card_table->byte_for(mr.start()); |
112 |
CardValue* last_byte = _card_table->byte_for(mr.last()); |
|
54006 | 113 |
// skip initial young cards |
49164
7e958a8ebcd3
8195142: Refactor out card table from CardTableModRefBS to flatten the BarrierSet hierarchy
eosterlund
parents:
49007
diff
changeset
|
114 |
for (; byte <= last_byte && *byte == G1CardTable::g1_young_card_val(); byte++); |
20403
45a89fbcd8f7
8014555: G1: Memory ordering problem with Conc refinement and card marking
mgerdin
parents:
20309
diff
changeset
|
115 |
|
42598
45562c0473fb
8166898: G1SATBCardTableLoggingModRefBS::invalidate() incorrect with whole_heap == true
lmesnik
parents:
40655
diff
changeset
|
116 |
if (byte <= last_byte) { |
45562c0473fb
8166898: G1SATBCardTableLoggingModRefBS::invalidate() incorrect with whole_heap == true
lmesnik
parents:
40655
diff
changeset
|
117 |
OrderAccess::storeload(); |
45562c0473fb
8166898: G1SATBCardTableLoggingModRefBS::invalidate() incorrect with whole_heap == true
lmesnik
parents:
40655
diff
changeset
|
118 |
// Enqueue if necessary. |
54006 | 119 |
Thread* thr = Thread::current(); |
120 |
G1DirtyCardQueue& queue = G1ThreadLocalData::dirty_card_queue(thr); |
|
121 |
for (; byte <= last_byte; byte++) { |
|
54110 | 122 |
CardValue bv = *byte; |
54006 | 123 |
if ((bv != G1CardTable::g1_young_card_val()) && |
124 |
(bv != G1CardTable::dirty_card_val())) { |
|
125 |
*byte = G1CardTable::dirty_card_val(); |
|
126 |
queue.enqueue(byte); |
|
1374 | 127 |
} |
128 |
} |
|
129 |
} |
|
130 |
} |
|
48961
120b61d50f85
8195103: Refactor out card table from CardTableModRefBS to flatten the BarrierSet hierarchy
eosterlund
parents:
48102
diff
changeset
|
131 |
|
49752
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49751
diff
changeset
|
132 |
void G1BarrierSet::on_thread_create(Thread* thread) { |
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49751
diff
changeset
|
133 |
// Create thread local data |
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49751
diff
changeset
|
134 |
G1ThreadLocalData::create(thread); |
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49751
diff
changeset
|
135 |
} |
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49751
diff
changeset
|
136 |
|
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49751
diff
changeset
|
137 |
void G1BarrierSet::on_thread_destroy(Thread* thread) { |
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49751
diff
changeset
|
138 |
// Destroy thread local data |
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49751
diff
changeset
|
139 |
G1ThreadLocalData::destroy(thread); |
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49751
diff
changeset
|
140 |
} |
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49751
diff
changeset
|
141 |
|
54006 | 142 |
void G1BarrierSet::on_thread_attach(Thread* thread) { |
49752
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49751
diff
changeset
|
143 |
assert(!G1ThreadLocalData::satb_mark_queue(thread).is_active(), "SATB queue should not be active"); |
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49751
diff
changeset
|
144 |
assert(G1ThreadLocalData::satb_mark_queue(thread).is_empty(), "SATB queue should be empty"); |
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49751
diff
changeset
|
145 |
assert(G1ThreadLocalData::dirty_card_queue(thread).is_active(), "Dirty card queue should be active"); |
54006 | 146 |
// Can't assert that the DCQ is empty. There is early execution on |
147 |
// the main thread, before it gets added to the threads list, which |
|
148 |
// is where this is called. That execution may enqueue dirty cards. |
|
49007
82d9d5744e5f
8197994: Move JavaThread::initialize_queues() logic to G1SATBCardTableLoggingModRefBS
pliden
parents:
48961
diff
changeset
|
149 |
|
82d9d5744e5f
8197994: Move JavaThread::initialize_queues() logic to G1SATBCardTableLoggingModRefBS
pliden
parents:
48961
diff
changeset
|
150 |
// If we are creating the thread during a marking cycle, we should |
54006 | 151 |
// set the active field of the SATB queue to true. That involves |
54366
2b48cedce327
8220671: Initialization race for non-JavaThread PtrQueues
kbarrett
parents:
54255
diff
changeset
|
152 |
// copying the global is_active value to this thread's queue. |
54006 | 153 |
bool is_satb_active = _satb_mark_queue_set.is_active(); |
154 |
G1ThreadLocalData::satb_mark_queue(thread).set_active(is_satb_active); |
|
49007
82d9d5744e5f
8197994: Move JavaThread::initialize_queues() logic to G1SATBCardTableLoggingModRefBS
pliden
parents:
48961
diff
changeset
|
155 |
} |
82d9d5744e5f
8197994: Move JavaThread::initialize_queues() logic to G1SATBCardTableLoggingModRefBS
pliden
parents:
48961
diff
changeset
|
156 |
|
54006 | 157 |
void G1BarrierSet::on_thread_detach(Thread* thread) { |
158 |
// Flush any deferred card marks. |
|
49455
848864ed9b17
8199604: Rename CardTableModRefBS to CardTableBarrierSet
eosterlund
parents:
49347
diff
changeset
|
159 |
CardTableBarrierSet::on_thread_detach(thread); |
49752
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49751
diff
changeset
|
160 |
G1ThreadLocalData::satb_mark_queue(thread).flush(); |
93d84f667d12
8201318: Introduce GCThreadLocalData to abstract GC-specific data belonging to a thread
pliden
parents:
49751
diff
changeset
|
161 |
G1ThreadLocalData::dirty_card_queue(thread).flush(); |
48961
120b61d50f85
8195103: Refactor out card table from CardTableModRefBS to flatten the BarrierSet hierarchy
eosterlund
parents:
48102
diff
changeset
|
162 |
} |