author | zgu |
Fri, 08 Nov 2019 09:51:18 -0500 | |
changeset 58986 | 66b1db608c12 |
parent 58985 | 5606867a5e6e |
permissions | -rw-r--r-- |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
1 |
/* |
54763
2584e5772546
8222926: Shenandoah build fails with --with-jvm-features=-compiler1
aoqi
parents:
54425
diff
changeset
|
2 |
* Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
3 |
* |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
4 |
* This code is free software; you can redistribute it and/or modify it |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
5 |
* under the terms of the GNU General Public License version 2 only, as |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
6 |
* published by the Free Software Foundation. |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
7 |
* |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
8 |
* This code is distributed in the hope that it will be useful, but WITHOUT |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
9 |
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
10 |
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
11 |
* version 2 for more details (a copy is included in the LICENSE file that |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
12 |
* accompanied this code). |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
13 |
* |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
14 |
* You should have received a copy of the GNU General Public License version |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
15 |
* 2 along with this work; if not, write to the Free Software Foundation, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
16 |
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
17 |
* |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
18 |
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
19 |
* or visit www.oracle.com if you need additional information or have any |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
20 |
* questions. |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
21 |
* |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
22 |
*/ |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
23 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
24 |
#include "precompiled.hpp" |
58985
5606867a5e6e
8233339: Shenandoah: Centralize load barrier decisions into ShenandoahBarrierSet
zgu
parents:
58946
diff
changeset
|
25 |
#include "gc/shenandoah/shenandoahBarrierSet.hpp" |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
26 |
#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" |
54766
1321f8cf9de5
8223567: Rename ShenandoahBrooksPointer to ShenandoahForwarding
rkennke
parents:
54763
diff
changeset
|
27 |
#include "gc/shenandoah/shenandoahForwarding.hpp" |
55718
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
28 |
#include "gc/shenandoah/shenandoahHeap.inline.hpp" |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
29 |
#include "gc/shenandoah/shenandoahHeapRegion.hpp" |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
30 |
#include "gc/shenandoah/shenandoahHeuristics.hpp" |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
31 |
#include "gc/shenandoah/shenandoahRuntime.hpp" |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
32 |
#include "gc/shenandoah/shenandoahThreadLocalData.hpp" |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
33 |
#include "interpreter/interpreter.hpp" |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
34 |
#include "interpreter/interp_masm.hpp" |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
35 |
#include "runtime/sharedRuntime.hpp" |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
36 |
#include "runtime/thread.hpp" |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
37 |
#ifdef COMPILER1 |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
38 |
#include "c1/c1_LIRAssembler.hpp" |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
39 |
#include "c1/c1_MacroAssembler.hpp" |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
40 |
#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp" |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
41 |
#endif |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
42 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
43 |
#define __ masm-> |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
44 |
|
54423 | 45 |
address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL; |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
46 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
47 |
void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, |
54979
f982c1a6582c
8224187: Refactor arraycopy_prologue to allow ZGC read barriers on arraycopy
smonteith
parents:
54766
diff
changeset
|
48 |
Register src, Register dst, Register count, RegSet saved_regs) { |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
49 |
if (is_oop) { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
50 |
bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; |
58218
0d7877278adf
8231086: Shenandoah: Stronger invariant for object-arraycopy
rkennke
parents:
57777
diff
changeset
|
51 |
if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) { |
54383 | 52 |
|
53 |
Label done; |
|
54 |
||
55 |
// Avoid calling runtime if count == 0 |
|
56 |
__ cbz(count, done); |
|
57 |
||
58 |
// Is marking active? |
|
59 |
Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); |
|
60 |
__ ldrb(rscratch1, gc_state); |
|
58218
0d7877278adf
8231086: Shenandoah: Stronger invariant for object-arraycopy
rkennke
parents:
57777
diff
changeset
|
61 |
if (dest_uninitialized) { |
58737
a39cdab8fac1
8232778: Shenandoah: SBSA::arraycopy_prologue checks wrong register
shade
parents:
58725
diff
changeset
|
62 |
__ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done); |
58218
0d7877278adf
8231086: Shenandoah: Stronger invariant for object-arraycopy
rkennke
parents:
57777
diff
changeset
|
63 |
} else { |
0d7877278adf
8231086: Shenandoah: Stronger invariant for object-arraycopy
rkennke
parents:
57777
diff
changeset
|
64 |
__ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING); |
0d7877278adf
8231086: Shenandoah: Stronger invariant for object-arraycopy
rkennke
parents:
57777
diff
changeset
|
65 |
__ tst(rscratch1, rscratch2); |
0d7877278adf
8231086: Shenandoah: Stronger invariant for object-arraycopy
rkennke
parents:
57777
diff
changeset
|
66 |
__ br(Assembler::EQ, done); |
0d7877278adf
8231086: Shenandoah: Stronger invariant for object-arraycopy
rkennke
parents:
57777
diff
changeset
|
67 |
} |
54383 | 68 |
|
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
69 |
__ push(saved_regs, sp); |
58218
0d7877278adf
8231086: Shenandoah: Stronger invariant for object-arraycopy
rkennke
parents:
57777
diff
changeset
|
70 |
if (UseCompressedOops) { |
0d7877278adf
8231086: Shenandoah: Stronger invariant for object-arraycopy
rkennke
parents:
57777
diff
changeset
|
71 |
if (dest_uninitialized) { |
0d7877278adf
8231086: Shenandoah: Stronger invariant for object-arraycopy
rkennke
parents:
57777
diff
changeset
|
72 |
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry), src, dst, count); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
73 |
} else { |
58218
0d7877278adf
8231086: Shenandoah: Stronger invariant for object-arraycopy
rkennke
parents:
57777
diff
changeset
|
74 |
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), src, dst, count); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
75 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
76 |
} else { |
58218
0d7877278adf
8231086: Shenandoah: Stronger invariant for object-arraycopy
rkennke
parents:
57777
diff
changeset
|
77 |
if (dest_uninitialized) { |
0d7877278adf
8231086: Shenandoah: Stronger invariant for object-arraycopy
rkennke
parents:
57777
diff
changeset
|
78 |
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry), src, dst, count); |
0d7877278adf
8231086: Shenandoah: Stronger invariant for object-arraycopy
rkennke
parents:
57777
diff
changeset
|
79 |
} else { |
0d7877278adf
8231086: Shenandoah: Stronger invariant for object-arraycopy
rkennke
parents:
57777
diff
changeset
|
80 |
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), src, dst, count); |
0d7877278adf
8231086: Shenandoah: Stronger invariant for object-arraycopy
rkennke
parents:
57777
diff
changeset
|
81 |
} |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
82 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
83 |
__ pop(saved_regs, sp); |
54383 | 84 |
__ bind(done); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
85 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
86 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
87 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
88 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
89 |
void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
90 |
Register obj, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
91 |
Register pre_val, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
92 |
Register thread, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
93 |
Register tmp, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
94 |
bool tosca_live, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
95 |
bool expand_call) { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
96 |
if (ShenandoahSATBBarrier) { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
97 |
satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
98 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
99 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
100 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
101 |
void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
102 |
Register obj, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
103 |
Register pre_val, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
104 |
Register thread, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
105 |
Register tmp, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
106 |
bool tosca_live, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
107 |
bool expand_call) { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
108 |
// If expand_call is true then we expand the call_VM_leaf macro |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
109 |
// directly to skip generating the check by |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
110 |
// InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp. |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
111 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
112 |
assert(thread == rthread, "must be"); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
113 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
114 |
Label done; |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
115 |
Label runtime; |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
116 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
117 |
assert_different_registers(obj, pre_val, tmp, rscratch1); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
118 |
assert(pre_val != noreg && tmp != noreg, "expecting a register"); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
119 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
120 |
Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset())); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
121 |
Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset())); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
122 |
Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
123 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
124 |
// Is marking active? |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
125 |
if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
126 |
__ ldrw(tmp, in_progress); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
127 |
} else { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
128 |
assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
129 |
__ ldrb(tmp, in_progress); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
130 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
131 |
__ cbzw(tmp, done); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
132 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
133 |
// Do we need to load the previous value? |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
134 |
if (obj != noreg) { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
135 |
__ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
136 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
137 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
138 |
// Is the previous value null? |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
139 |
__ cbz(pre_val, done); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
140 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
141 |
// Can we store original value in the thread's buffer? |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
142 |
// Is index == 0? |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
143 |
// (The index field is typed as size_t.) |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
144 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
145 |
__ ldr(tmp, index); // tmp := *index_adr |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
146 |
__ cbz(tmp, runtime); // tmp == 0? |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
147 |
// If yes, goto runtime |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
148 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
149 |
__ sub(tmp, tmp, wordSize); // tmp := tmp - wordSize |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
150 |
__ str(tmp, index); // *index_adr := tmp |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
151 |
__ ldr(rscratch1, buffer); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
152 |
__ add(tmp, tmp, rscratch1); // tmp := tmp + *buffer_adr |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
153 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
154 |
// Record the previous value |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
155 |
__ str(pre_val, Address(tmp, 0)); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
156 |
__ b(done); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
157 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
158 |
__ bind(runtime); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
159 |
// save the live input values |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
160 |
RegSet saved = RegSet::of(pre_val); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
161 |
if (tosca_live) saved += RegSet::of(r0); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
162 |
if (obj != noreg) saved += RegSet::of(obj); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
163 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
164 |
__ push(saved, sp); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
165 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
166 |
// Calling the runtime using the regular call_VM_leaf mechanism generates |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
167 |
// code (generated by InterpreterMacroAssember::call_VM_leaf_base) |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
168 |
// that checks that the *(rfp+frame::interpreter_frame_last_sp) == NULL. |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
169 |
// |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
170 |
// If we care generating the pre-barrier without a frame (e.g. in the |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
171 |
// intrinsified Reference.get() routine) then ebp might be pointing to |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
172 |
// the caller frame and so this check will most likely fail at runtime. |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
173 |
// |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
174 |
// Expanding the call directly bypasses the generation of the check. |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
175 |
// So when we do not have have a full interpreter frame on the stack |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
176 |
// expand_call should be passed true. |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
177 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
178 |
if (expand_call) { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
179 |
assert(pre_val != c_rarg1, "smashed arg"); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
180 |
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
181 |
} else { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
182 |
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
183 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
184 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
185 |
__ pop(saved, sp); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
186 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
187 |
__ bind(done); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
188 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
189 |
|
55076
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
190 |
void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) { |
54423 | 191 |
assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled"); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
192 |
Label is_null; |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
193 |
__ cbz(dst, is_null); |
55076
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
194 |
resolve_forward_pointer_not_null(masm, dst, tmp); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
195 |
__ bind(is_null); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
196 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
197 |
|
55076
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
198 |
// IMPORTANT: This must preserve all registers, even rscratch1 and rscratch2, except those explicitely |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
199 |
// passed in. |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
200 |
void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp) { |
54423 | 201 |
assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled"); |
55076
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
202 |
// The below loads the mark word, checks if the lowest two bits are |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
203 |
// set, and if so, clear the lowest two bits and copy the result |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
204 |
// to dst. Otherwise it leaves dst alone. |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
205 |
// Implementing this is surprisingly awkward. I do it here by: |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
206 |
// - Inverting the mark word |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
207 |
// - Test lowest two bits == 0 |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
208 |
// - If so, set the lowest two bits |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
209 |
// - Invert the result back, and copy to dst |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
210 |
|
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
211 |
bool borrow_reg = (tmp == noreg); |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
212 |
if (borrow_reg) { |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
213 |
// No free registers available. Make one useful. |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
214 |
tmp = rscratch1; |
58407
b14643d898d3
8231583: Shenandoah: Fix register clash in SBSA::resolve_forwarding_pointer() borrowing
shade
parents:
58273
diff
changeset
|
215 |
if (tmp == dst) { |
b14643d898d3
8231583: Shenandoah: Fix register clash in SBSA::resolve_forwarding_pointer() borrowing
shade
parents:
58273
diff
changeset
|
216 |
tmp = rscratch2; |
b14643d898d3
8231583: Shenandoah: Fix register clash in SBSA::resolve_forwarding_pointer() borrowing
shade
parents:
58273
diff
changeset
|
217 |
} |
55076
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
218 |
__ push(RegSet::of(tmp), sp); |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
219 |
} |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
220 |
|
58407
b14643d898d3
8231583: Shenandoah: Fix register clash in SBSA::resolve_forwarding_pointer() borrowing
shade
parents:
58273
diff
changeset
|
221 |
assert_different_registers(tmp, dst); |
b14643d898d3
8231583: Shenandoah: Fix register clash in SBSA::resolve_forwarding_pointer() borrowing
shade
parents:
58273
diff
changeset
|
222 |
|
55076
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
223 |
Label done; |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
224 |
__ ldr(tmp, Address(dst, oopDesc::mark_offset_in_bytes())); |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
225 |
__ eon(tmp, tmp, zr); |
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
57716
diff
changeset
|
226 |
__ ands(zr, tmp, markWord::lock_mask_in_place); |
55076
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
227 |
__ br(Assembler::NE, done); |
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
57716
diff
changeset
|
228 |
__ orr(tmp, tmp, markWord::marked_value); |
55076
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
229 |
__ eon(dst, tmp, zr); |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
230 |
__ bind(done); |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
231 |
|
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
232 |
if (borrow_reg) { |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
233 |
__ pop(RegSet::of(tmp), sp); |
785a12e0f89b
8224584: Shenandoah: Eliminate forwarding pointer word
rkennke
parents:
54979
diff
changeset
|
234 |
} |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
235 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
236 |
|
58819
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
237 |
void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Address load_addr) { |
54423 | 238 |
assert(ShenandoahLoadRefBarrier, "Should be enabled"); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
239 |
assert(dst != rscratch2, "need rscratch2"); |
58819
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
240 |
assert_different_registers(load_addr.base(), load_addr.index(), rscratch1, rscratch2); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
241 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
242 |
Label done; |
54423 | 243 |
__ enter(); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
244 |
Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); |
54423 | 245 |
__ ldrb(rscratch2, gc_state); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
246 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
247 |
// Check for heap stability |
54423 | 248 |
__ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, done); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
249 |
|
58819
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
250 |
// use r1 for load address |
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
251 |
Register result_dst = dst; |
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
252 |
if (dst == r1) { |
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
253 |
__ mov(rscratch1, dst); |
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
254 |
dst = rscratch1; |
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
255 |
} |
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
256 |
|
58986
66b1db608c12
8233337: Shenandoah: Cleanup AArch64 SBSA::load_reference_barrier_not_null()
zgu
parents:
58985
diff
changeset
|
257 |
// Save r0 and r1, unless it is an output register |
66b1db608c12
8233337: Shenandoah: Cleanup AArch64 SBSA::load_reference_barrier_not_null()
zgu
parents:
58985
diff
changeset
|
258 |
RegSet to_save = RegSet::of(r0, r1) - result_dst; |
66b1db608c12
8233337: Shenandoah: Cleanup AArch64 SBSA::load_reference_barrier_not_null()
zgu
parents:
58985
diff
changeset
|
259 |
__ push(to_save, sp); |
58819
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
260 |
__ lea(r1, load_addr); |
58986
66b1db608c12
8233337: Shenandoah: Cleanup AArch64 SBSA::load_reference_barrier_not_null()
zgu
parents:
58985
diff
changeset
|
261 |
__ mov(r0, dst); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
262 |
|
54423 | 263 |
__ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb()))); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
264 |
|
58986
66b1db608c12
8233337: Shenandoah: Cleanup AArch64 SBSA::load_reference_barrier_not_null()
zgu
parents:
58985
diff
changeset
|
265 |
__ mov(result_dst, r0); |
66b1db608c12
8233337: Shenandoah: Cleanup AArch64 SBSA::load_reference_barrier_not_null()
zgu
parents:
58985
diff
changeset
|
266 |
__ pop(to_save, sp); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
267 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
268 |
__ bind(done); |
54423 | 269 |
__ leave(); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
270 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
271 |
|
58725
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
272 |
void ShenandoahBarrierSetAssembler::load_reference_barrier_native(MacroAssembler* masm, Register dst, Address load_addr) { |
55718
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
273 |
if (!ShenandoahLoadRefBarrier) { |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
274 |
return; |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
275 |
} |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
276 |
|
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
277 |
assert(dst != rscratch2, "need rscratch2"); |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
278 |
|
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
279 |
Label is_null; |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
280 |
Label done; |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
281 |
|
58725
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
282 |
__ block_comment("load_reference_barrier_native { "); |
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
283 |
|
55718
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
284 |
__ cbz(dst, is_null); |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
285 |
|
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
286 |
__ enter(); |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
287 |
|
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
288 |
Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
289 |
__ ldrb(rscratch2, gc_state); |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
290 |
|
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
291 |
// Check for heap in evacuation phase |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
292 |
__ tbz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, done); |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
293 |
|
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
294 |
__ mov(rscratch2, dst); |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
295 |
__ push_call_clobbered_registers(); |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
296 |
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native)); |
58725
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
297 |
__ lea(r1, load_addr); |
55718
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
298 |
__ mov(r0, rscratch2); |
57565 | 299 |
__ blr(lr); |
55718
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
300 |
__ mov(rscratch2, r0); |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
301 |
__ pop_call_clobbered_registers(); |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
302 |
__ mov(dst, rscratch2); |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
303 |
|
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
304 |
__ bind(done); |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
305 |
__ leave(); |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
306 |
__ bind(is_null); |
58725
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
307 |
__ block_comment("} load_reference_barrier_native"); |
55718
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
308 |
} |
9fbc32392798
8227635: Shenandoah: SHBSA::load_at() needs to deal IN_NATIVE load
zgu
parents:
55679
diff
changeset
|
309 |
|
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
310 |
void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
311 |
if (ShenandoahStoreValEnqueueBarrier) { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
312 |
// Save possibly live regs. |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
313 |
RegSet live_regs = RegSet::range(r0, r4) - dst; |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
314 |
__ push(live_regs, sp); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
315 |
__ strd(v0, __ pre(sp, 2 * -wordSize)); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
316 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
317 |
satb_write_barrier_pre(masm, noreg, dst, rthread, tmp, true, false); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
318 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
319 |
// Restore possibly live regs. |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
320 |
__ ldrd(v0, __ post(sp, 2 * wordSize)); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
321 |
__ pop(live_regs, sp); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
322 |
} |
54423 | 323 |
} |
324 |
||
58819
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
325 |
void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr) { |
54423 | 326 |
if (ShenandoahLoadRefBarrier) { |
327 |
Label is_null; |
|
328 |
__ cbz(dst, is_null); |
|
58819
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
329 |
load_reference_barrier_not_null(masm, dst, load_addr); |
54423 | 330 |
__ bind(is_null); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
331 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
332 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
333 |
|
58946
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
334 |
// |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
335 |
// Arguments: |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
336 |
// |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
337 |
// Inputs: |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
338 |
// src: oop location to load from, might be clobbered |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
339 |
// |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
340 |
// Output: |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
341 |
// dst: oop loaded from src location |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
342 |
// |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
343 |
// Kill: |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
344 |
// rscratch1 (scratch reg) |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
345 |
// |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
346 |
// Alias: |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
347 |
// dst: rscratch1 (might use rscratch1 as temporary output register to avoid clobbering src) |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
348 |
// |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
349 |
void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
350 |
Register dst, Address src, Register tmp1, Register tmp_thread) { |
58946
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
351 |
// 1: non-reference load, no additional barrier is needed |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
352 |
if (!is_reference_type(type)) { |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
353 |
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
354 |
return; |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
355 |
} |
58725
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
356 |
|
58985
5606867a5e6e
8233339: Shenandoah: Centralize load barrier decisions into ShenandoahBarrierSet
zgu
parents:
58946
diff
changeset
|
357 |
// 2: load a reference from src location and apply LRB if needed |
5606867a5e6e
8233339: Shenandoah: Centralize load barrier decisions into ShenandoahBarrierSet
zgu
parents:
58946
diff
changeset
|
358 |
if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) { |
58946
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
359 |
Register result_dst = dst; |
58725
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
360 |
|
58946
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
361 |
// Preserve src location for LRB |
58725
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
362 |
if (dst == src.base() || dst == src.index()) { |
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
363 |
dst = rscratch1; |
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
364 |
} |
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
365 |
assert_different_registers(dst, src.base(), src.index()); |
58946
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
366 |
|
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
367 |
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
368 |
|
58985
5606867a5e6e
8233339: Shenandoah: Centralize load barrier decisions into ShenandoahBarrierSet
zgu
parents:
58946
diff
changeset
|
369 |
if (ShenandoahBarrierSet::use_load_reference_barrier_native(decorators, type)) { |
58725
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
370 |
load_reference_barrier_native(masm, dst, src); |
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
371 |
} else { |
58819
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
372 |
load_reference_barrier(masm, dst, src); |
58725
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
373 |
} |
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
374 |
|
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
375 |
if (dst != result_dst) { |
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
376 |
__ mov(result_dst, dst); |
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
377 |
dst = result_dst; |
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
378 |
} |
58946
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
379 |
} else { |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
380 |
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); |
83810b7d12e7
8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
zgu
parents:
58916
diff
changeset
|
381 |
} |
58725
190ec5fae2af
8232712: Shenandoah: SIGBUS in load_reference_barrier_native
zgu
parents:
58407
diff
changeset
|
382 |
|
58985
5606867a5e6e
8233339: Shenandoah: Centralize load barrier decisions into ShenandoahBarrierSet
zgu
parents:
58946
diff
changeset
|
383 |
// 3: apply keep-alive barrier if needed |
5606867a5e6e
8233339: Shenandoah: Centralize load barrier decisions into ShenandoahBarrierSet
zgu
parents:
58946
diff
changeset
|
384 |
if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) { |
5606867a5e6e
8233339: Shenandoah: Centralize load barrier decisions into ShenandoahBarrierSet
zgu
parents:
58946
diff
changeset
|
385 |
__ enter(); |
5606867a5e6e
8233339: Shenandoah: Centralize load barrier decisions into ShenandoahBarrierSet
zgu
parents:
58946
diff
changeset
|
386 |
satb_write_barrier_pre(masm /* masm */, |
5606867a5e6e
8233339: Shenandoah: Centralize load barrier decisions into ShenandoahBarrierSet
zgu
parents:
58946
diff
changeset
|
387 |
noreg /* obj */, |
5606867a5e6e
8233339: Shenandoah: Centralize load barrier decisions into ShenandoahBarrierSet
zgu
parents:
58946
diff
changeset
|
388 |
dst /* pre_val */, |
5606867a5e6e
8233339: Shenandoah: Centralize load barrier decisions into ShenandoahBarrierSet
zgu
parents:
58946
diff
changeset
|
389 |
rthread /* thread */, |
5606867a5e6e
8233339: Shenandoah: Centralize load barrier decisions into ShenandoahBarrierSet
zgu
parents:
58946
diff
changeset
|
390 |
tmp1 /* tmp */, |
5606867a5e6e
8233339: Shenandoah: Centralize load barrier decisions into ShenandoahBarrierSet
zgu
parents:
58946
diff
changeset
|
391 |
true /* tosca_live */, |
5606867a5e6e
8233339: Shenandoah: Centralize load barrier decisions into ShenandoahBarrierSet
zgu
parents:
58946
diff
changeset
|
392 |
true /* expand_call */); |
5606867a5e6e
8233339: Shenandoah: Centralize load barrier decisions into ShenandoahBarrierSet
zgu
parents:
58946
diff
changeset
|
393 |
__ leave(); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
394 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
395 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
396 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
397 |
void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
398 |
Address dst, Register val, Register tmp1, Register tmp2) { |
58273
08a5148e7c4e
8230505: Replace JVM type comparisons to T_OBJECT and T_ARRAY with call to is_reference_type
lfoltan
parents:
58219
diff
changeset
|
399 |
bool on_oop = is_reference_type(type); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
400 |
if (!on_oop) { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
401 |
BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
402 |
return; |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
403 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
404 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
405 |
// flatten object address if needed |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
406 |
if (dst.index() == noreg && dst.offset() == 0) { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
407 |
if (dst.base() != r3) { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
408 |
__ mov(r3, dst.base()); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
409 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
410 |
} else { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
411 |
__ lea(r3, dst); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
412 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
413 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
414 |
shenandoah_write_barrier_pre(masm, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
415 |
r3 /* obj */, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
416 |
tmp2 /* pre_val */, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
417 |
rthread /* thread */, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
418 |
tmp1 /* tmp */, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
419 |
val != noreg /* tosca_live */, |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
420 |
false /* expand_call */); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
421 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
422 |
if (val == noreg) { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
423 |
BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), noreg, noreg, noreg); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
424 |
} else { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
425 |
storeval_barrier(masm, val, tmp1); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
426 |
// G1 barrier needs uncompressed oop for region cross check. |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
427 |
Register new_val = val; |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
428 |
if (UseCompressedOops) { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
429 |
new_val = rscratch2; |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
430 |
__ mov(new_val, val); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
431 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
432 |
BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
433 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
434 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
435 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
436 |
|
57542
db740ced41c4
8228532: Shenandoah: Implement SBSA::try_resolve_jobject_in_native()
zgu
parents:
55735
diff
changeset
|
437 |
void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env, |
db740ced41c4
8228532: Shenandoah: Implement SBSA::try_resolve_jobject_in_native()
zgu
parents:
55735
diff
changeset
|
438 |
Register obj, Register tmp, Label& slowpath) { |
db740ced41c4
8228532: Shenandoah: Implement SBSA::try_resolve_jobject_in_native()
zgu
parents:
55735
diff
changeset
|
439 |
Label done; |
db740ced41c4
8228532: Shenandoah: Implement SBSA::try_resolve_jobject_in_native()
zgu
parents:
55735
diff
changeset
|
440 |
// Resolve jobject |
db740ced41c4
8228532: Shenandoah: Implement SBSA::try_resolve_jobject_in_native()
zgu
parents:
55735
diff
changeset
|
441 |
BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath); |
db740ced41c4
8228532: Shenandoah: Implement SBSA::try_resolve_jobject_in_native()
zgu
parents:
55735
diff
changeset
|
442 |
|
db740ced41c4
8228532: Shenandoah: Implement SBSA::try_resolve_jobject_in_native()
zgu
parents:
55735
diff
changeset
|
443 |
// Check for null. |
db740ced41c4
8228532: Shenandoah: Implement SBSA::try_resolve_jobject_in_native()
zgu
parents:
55735
diff
changeset
|
444 |
__ cbz(obj, done); |
db740ced41c4
8228532: Shenandoah: Implement SBSA::try_resolve_jobject_in_native()
zgu
parents:
55735
diff
changeset
|
445 |
|
db740ced41c4
8228532: Shenandoah: Implement SBSA::try_resolve_jobject_in_native()
zgu
parents:
55735
diff
changeset
|
446 |
assert(obj != rscratch2, "need rscratch2"); |
58916
7298f7c503b4
8233232: AArch64: Shenandoah: jni_fast_GetLongField is broken
aph
parents:
58858
diff
changeset
|
447 |
Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset()); |
7298f7c503b4
8233232: AArch64: Shenandoah: jni_fast_GetLongField is broken
aph
parents:
58858
diff
changeset
|
448 |
__ lea(rscratch2, gc_state); |
7298f7c503b4
8233232: AArch64: Shenandoah: jni_fast_GetLongField is broken
aph
parents:
58858
diff
changeset
|
449 |
__ ldrb(rscratch2, Address(rscratch2)); |
57542
db740ced41c4
8228532: Shenandoah: Implement SBSA::try_resolve_jobject_in_native()
zgu
parents:
55735
diff
changeset
|
450 |
|
db740ced41c4
8228532: Shenandoah: Implement SBSA::try_resolve_jobject_in_native()
zgu
parents:
55735
diff
changeset
|
451 |
// Check for heap in evacuation phase |
db740ced41c4
8228532: Shenandoah: Implement SBSA::try_resolve_jobject_in_native()
zgu
parents:
55735
diff
changeset
|
452 |
__ tbnz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, slowpath); |
db740ced41c4
8228532: Shenandoah: Implement SBSA::try_resolve_jobject_in_native()
zgu
parents:
55735
diff
changeset
|
453 |
|
db740ced41c4
8228532: Shenandoah: Implement SBSA::try_resolve_jobject_in_native()
zgu
parents:
55735
diff
changeset
|
454 |
__ bind(done); |
db740ced41c4
8228532: Shenandoah: Implement SBSA::try_resolve_jobject_in_native()
zgu
parents:
55735
diff
changeset
|
455 |
} |
db740ced41c4
8228532: Shenandoah: Implement SBSA::try_resolve_jobject_in_native()
zgu
parents:
55735
diff
changeset
|
456 |
|
db740ced41c4
8228532: Shenandoah: Implement SBSA::try_resolve_jobject_in_native()
zgu
parents:
55735
diff
changeset
|
457 |
|
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
458 |
void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val, |
53519
74a5ef4c81cc
8217016: Shenandoah: Streamline generation of CAS barriers
rkennke
parents:
52925
diff
changeset
|
459 |
bool acquire, bool release, bool weak, bool is_cae, |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
460 |
Register result) { |
53719
3a56e823d843
8217874: Shenandoah: Clobbered register in ShenandoahBarrierSetAssembler::cmpxchg_oop()
rkennke
parents:
53519
diff
changeset
|
461 |
Register tmp1 = rscratch1; |
3a56e823d843
8217874: Shenandoah: Clobbered register in ShenandoahBarrierSetAssembler::cmpxchg_oop()
rkennke
parents:
53519
diff
changeset
|
462 |
Register tmp2 = rscratch2; |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
463 |
bool is_narrow = UseCompressedOops; |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
464 |
Assembler::operand_size size = is_narrow ? Assembler::word : Assembler::xword; |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
465 |
|
53719
3a56e823d843
8217874: Shenandoah: Clobbered register in ShenandoahBarrierSetAssembler::cmpxchg_oop()
rkennke
parents:
53519
diff
changeset
|
466 |
assert_different_registers(addr, expected, new_val, tmp1, tmp2); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
467 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
468 |
Label retry, done, fail; |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
469 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
470 |
// CAS, using LL/SC pair. |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
471 |
__ bind(retry); |
53719
3a56e823d843
8217874: Shenandoah: Clobbered register in ShenandoahBarrierSetAssembler::cmpxchg_oop()
rkennke
parents:
53519
diff
changeset
|
472 |
__ load_exclusive(tmp1, addr, size, acquire); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
473 |
if (is_narrow) { |
53719
3a56e823d843
8217874: Shenandoah: Clobbered register in ShenandoahBarrierSetAssembler::cmpxchg_oop()
rkennke
parents:
53519
diff
changeset
|
474 |
__ cmpw(tmp1, expected); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
475 |
} else { |
53719
3a56e823d843
8217874: Shenandoah: Clobbered register in ShenandoahBarrierSetAssembler::cmpxchg_oop()
rkennke
parents:
53519
diff
changeset
|
476 |
__ cmp(tmp1, expected); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
477 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
478 |
__ br(Assembler::NE, fail); |
53719
3a56e823d843
8217874: Shenandoah: Clobbered register in ShenandoahBarrierSetAssembler::cmpxchg_oop()
rkennke
parents:
53519
diff
changeset
|
479 |
__ store_exclusive(tmp2, new_val, addr, size, release); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
480 |
if (weak) { |
53719
3a56e823d843
8217874: Shenandoah: Clobbered register in ShenandoahBarrierSetAssembler::cmpxchg_oop()
rkennke
parents:
53519
diff
changeset
|
481 |
__ cmpw(tmp2, 0u); // If the store fails, return NE to our caller |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
482 |
} else { |
53719
3a56e823d843
8217874: Shenandoah: Clobbered register in ShenandoahBarrierSetAssembler::cmpxchg_oop()
rkennke
parents:
53519
diff
changeset
|
483 |
__ cbnzw(tmp2, retry); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
484 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
485 |
__ b(done); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
486 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
487 |
__ bind(fail); |
53719
3a56e823d843
8217874: Shenandoah: Clobbered register in ShenandoahBarrierSetAssembler::cmpxchg_oop()
rkennke
parents:
53519
diff
changeset
|
488 |
// Check if rb(expected)==rb(tmp1) |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
489 |
// Shuffle registers so that we have memory value ready for next expected. |
53719
3a56e823d843
8217874: Shenandoah: Clobbered register in ShenandoahBarrierSetAssembler::cmpxchg_oop()
rkennke
parents:
53519
diff
changeset
|
490 |
__ mov(tmp2, expected); |
3a56e823d843
8217874: Shenandoah: Clobbered register in ShenandoahBarrierSetAssembler::cmpxchg_oop()
rkennke
parents:
53519
diff
changeset
|
491 |
__ mov(expected, tmp1); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
492 |
if (is_narrow) { |
53719
3a56e823d843
8217874: Shenandoah: Clobbered register in ShenandoahBarrierSetAssembler::cmpxchg_oop()
rkennke
parents:
53519
diff
changeset
|
493 |
__ decode_heap_oop(tmp1, tmp1); |
3a56e823d843
8217874: Shenandoah: Clobbered register in ShenandoahBarrierSetAssembler::cmpxchg_oop()
rkennke
parents:
53519
diff
changeset
|
494 |
__ decode_heap_oop(tmp2, tmp2); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
495 |
} |
54423 | 496 |
resolve_forward_pointer(masm, tmp1); |
497 |
resolve_forward_pointer(masm, tmp2); |
|
53719
3a56e823d843
8217874: Shenandoah: Clobbered register in ShenandoahBarrierSetAssembler::cmpxchg_oop()
rkennke
parents:
53519
diff
changeset
|
498 |
__ cmp(tmp1, tmp2); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
499 |
// Retry with expected now being the value we just loaded from addr. |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
500 |
__ br(Assembler::EQ, retry); |
53519
74a5ef4c81cc
8217016: Shenandoah: Streamline generation of CAS barriers
rkennke
parents:
52925
diff
changeset
|
501 |
if (is_cae && is_narrow) { |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
502 |
// For cmp-and-exchange and narrow oops, we need to restore |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
503 |
// the compressed old-value. We moved it to 'expected' a few lines up. |
53719
3a56e823d843
8217874: Shenandoah: Clobbered register in ShenandoahBarrierSetAssembler::cmpxchg_oop()
rkennke
parents:
53519
diff
changeset
|
504 |
__ mov(tmp1, expected); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
505 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
506 |
__ bind(done); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
507 |
|
53719
3a56e823d843
8217874: Shenandoah: Clobbered register in ShenandoahBarrierSetAssembler::cmpxchg_oop()
rkennke
parents:
53519
diff
changeset
|
508 |
if (is_cae) { |
3a56e823d843
8217874: Shenandoah: Clobbered register in ShenandoahBarrierSetAssembler::cmpxchg_oop()
rkennke
parents:
53519
diff
changeset
|
509 |
__ mov(result, tmp1); |
3a56e823d843
8217874: Shenandoah: Clobbered register in ShenandoahBarrierSetAssembler::cmpxchg_oop()
rkennke
parents:
53519
diff
changeset
|
510 |
} else { |
53519
74a5ef4c81cc
8217016: Shenandoah: Streamline generation of CAS barriers
rkennke
parents:
52925
diff
changeset
|
511 |
__ cset(result, Assembler::EQ); |
74a5ef4c81cc
8217016: Shenandoah: Streamline generation of CAS barriers
rkennke
parents:
52925
diff
changeset
|
512 |
} |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
513 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
514 |
|
54763
2584e5772546
8222926: Shenandoah build fails with --with-jvm-features=-compiler1
aoqi
parents:
54425
diff
changeset
|
515 |
#undef __ |
2584e5772546
8222926: Shenandoah build fails with --with-jvm-features=-compiler1
aoqi
parents:
54425
diff
changeset
|
516 |
|
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
517 |
#ifdef COMPILER1 |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
518 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
519 |
#define __ ce->masm()-> |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
520 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
521 |
void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
522 |
ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
523 |
// At this point we know that marking is in progress. |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
524 |
// If do_load() is true then we have to emit the |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
525 |
// load of the previous value; otherwise it has already |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
526 |
// been loaded into _pre_val. |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
527 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
528 |
__ bind(*stub->entry()); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
529 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
530 |
assert(stub->pre_val()->is_register(), "Precondition."); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
531 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
532 |
Register pre_val_reg = stub->pre_val()->as_register(); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
533 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
534 |
if (stub->do_load()) { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
535 |
ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
536 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
537 |
__ cbz(pre_val_reg, *stub->continuation()); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
538 |
ce->store_parameter(stub->pre_val()->as_register(), 0); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
539 |
__ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
540 |
__ b(*stub->continuation()); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
541 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
542 |
|
54423 | 543 |
void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) { |
57716 | 544 |
ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); |
545 |
__ bind(*stub->entry()); |
|
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
546 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
547 |
Register obj = stub->obj()->as_register(); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
548 |
Register res = stub->result()->as_register(); |
58858
75099fcf7962
8233165: Shenandoah:SBSA::gen_load_reference_barrier_stub() should use pointer register for address on aarch64
zgu
parents:
58819
diff
changeset
|
549 |
Register addr = stub->addr()->as_pointer_register(); |
57716 | 550 |
Register tmp1 = stub->tmp1()->as_register(); |
551 |
Register tmp2 = stub->tmp2()->as_register(); |
|
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
552 |
|
57716 | 553 |
assert(res == r0, "result must arrive in r0"); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
554 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
555 |
if (res != obj) { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
556 |
__ mov(res, obj); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
557 |
} |
57716 | 558 |
|
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
559 |
// Check for null. |
57716 | 560 |
__ cbz(res, *stub->continuation()); |
561 |
||
562 |
// Check for object in cset. |
|
563 |
__ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr()); |
|
564 |
__ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint()); |
|
565 |
__ ldrb(tmp2, Address(tmp2, tmp1)); |
|
566 |
__ cbz(tmp2, *stub->continuation()); |
|
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
567 |
|
57716 | 568 |
// Check if object is already forwarded. |
569 |
Label slow_path; |
|
570 |
__ ldr(tmp1, Address(res, oopDesc::mark_offset_in_bytes())); |
|
571 |
__ eon(tmp1, tmp1, zr); |
|
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
57716
diff
changeset
|
572 |
__ ands(zr, tmp1, markWord::lock_mask_in_place); |
57716 | 573 |
__ br(Assembler::NE, slow_path); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
574 |
|
57716 | 575 |
// Decode forwarded object. |
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
57716
diff
changeset
|
576 |
__ orr(tmp1, tmp1, markWord::marked_value); |
57716 | 577 |
__ eon(res, tmp1, zr); |
578 |
__ b(*stub->continuation()); |
|
579 |
||
580 |
__ bind(slow_path); |
|
581 |
ce->store_parameter(res, 0); |
|
58219
bc0648405d67
8231087: Shenandoah: Self-fixing load reference barriers for C1/C2
rkennke
parents:
58218
diff
changeset
|
582 |
ce->store_parameter(addr, 1); |
57716 | 583 |
__ far_call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin())); |
584 |
||
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
585 |
__ b(*stub->continuation()); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
586 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
587 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
588 |
#undef __ |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
589 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
590 |
#define __ sasm-> |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
591 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
592 |
void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
593 |
__ prologue("shenandoah_pre_barrier", false); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
594 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
595 |
// arg0 : previous value of memory |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
596 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
597 |
BarrierSet* bs = BarrierSet::barrier_set(); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
598 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
599 |
const Register pre_val = r0; |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
600 |
const Register thread = rthread; |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
601 |
const Register tmp = rscratch1; |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
602 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
603 |
Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset())); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
604 |
Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
605 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
606 |
Label done; |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
607 |
Label runtime; |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
608 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
609 |
// Is marking still active? |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
610 |
Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
611 |
__ ldrb(tmp, gc_state); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
612 |
__ mov(rscratch2, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
613 |
__ tst(tmp, rscratch2); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
614 |
__ br(Assembler::EQ, done); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
615 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
616 |
// Can we store original value in the thread's buffer? |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
617 |
__ ldr(tmp, queue_index); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
618 |
__ cbz(tmp, runtime); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
619 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
620 |
__ sub(tmp, tmp, wordSize); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
621 |
__ str(tmp, queue_index); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
622 |
__ ldr(rscratch2, buffer); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
623 |
__ add(tmp, tmp, rscratch2); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
624 |
__ load_parameter(0, rscratch2); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
625 |
__ str(rscratch2, Address(tmp, 0)); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
626 |
__ b(done); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
627 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
628 |
__ bind(runtime); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
629 |
__ push_call_clobbered_registers(); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
630 |
__ load_parameter(0, pre_val); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
631 |
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
632 |
__ pop_call_clobbered_registers(); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
633 |
__ bind(done); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
634 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
635 |
__ epilogue(); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
636 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
637 |
|
57716 | 638 |
void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm) { |
639 |
__ prologue("shenandoah_load_reference_barrier", false); |
|
640 |
// arg0 : object to be resolved |
|
641 |
||
642 |
__ push_call_clobbered_registers(); |
|
643 |
__ load_parameter(0, r0); |
|
58219
bc0648405d67
8231087: Shenandoah: Self-fixing load reference barriers for C1/C2
rkennke
parents:
58218
diff
changeset
|
644 |
__ load_parameter(1, r1); |
bc0648405d67
8231087: Shenandoah: Self-fixing load reference barriers for C1/C2
rkennke
parents:
58218
diff
changeset
|
645 |
if (UseCompressedOops) { |
58819
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
646 |
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow)); |
58219
bc0648405d67
8231087: Shenandoah: Self-fixing load reference barriers for C1/C2
rkennke
parents:
58218
diff
changeset
|
647 |
} else { |
58819
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
648 |
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier)); |
58219
bc0648405d67
8231087: Shenandoah: Self-fixing load reference barriers for C1/C2
rkennke
parents:
58218
diff
changeset
|
649 |
} |
57716 | 650 |
__ blr(lr); |
651 |
__ mov(rscratch1, r0); |
|
652 |
__ pop_call_clobbered_registers(); |
|
653 |
__ mov(r0, rscratch1); |
|
654 |
||
655 |
__ epilogue(); |
|
656 |
} |
|
657 |
||
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
658 |
#undef __ |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
659 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
660 |
#endif // COMPILER1 |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
661 |
|
54423 | 662 |
address ShenandoahBarrierSetAssembler::shenandoah_lrb() { |
663 |
assert(_shenandoah_lrb != NULL, "need load reference barrier stub"); |
|
664 |
return _shenandoah_lrb; |
|
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
665 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
666 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
667 |
#define __ cgen->assembler()-> |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
668 |
|
54423 | 669 |
// Shenandoah load reference barrier. |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
670 |
// |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
671 |
// Input: |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
672 |
// r0: OOP to evacuate. Not null. |
58819
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
673 |
// r1: load address |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
674 |
// |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
675 |
// Output: |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
676 |
// r0: Pointer to evacuated OOP. |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
677 |
// |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
678 |
// Trash rscratch1, rscratch2. Preserve everything else. |
54423 | 679 |
address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) { |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
680 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
681 |
__ align(6); |
54423 | 682 |
StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb"); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
683 |
address start = __ pc(); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
684 |
|
54423 | 685 |
Label work, done; |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
686 |
__ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr()); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
687 |
__ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint()); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
688 |
__ ldrb(rscratch2, Address(rscratch2, rscratch1)); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
689 |
__ tbnz(rscratch2, 0, work); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
690 |
__ ret(lr); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
691 |
__ bind(work); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
692 |
|
57716 | 693 |
Label slow_path; |
694 |
__ ldr(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes())); |
|
695 |
__ eon(rscratch1, rscratch1, zr); |
|
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
57716
diff
changeset
|
696 |
__ ands(zr, rscratch1, markWord::lock_mask_in_place); |
57716 | 697 |
__ br(Assembler::NE, slow_path); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
698 |
|
57716 | 699 |
// Decode forwarded object. |
57777
90ead0febf56
8229258: Rework markOop and markOopDesc into a simpler mark word value carrier
stefank
parents:
57716
diff
changeset
|
700 |
__ orr(rscratch1, rscratch1, markWord::marked_value); |
57716 | 701 |
__ eon(r0, rscratch1, zr); |
702 |
__ ret(lr); |
|
703 |
||
704 |
__ bind(slow_path); |
|
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
705 |
__ enter(); // required for proper stackwalking of RuntimeStub frame |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
706 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
707 |
__ push_call_clobbered_registers(); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
708 |
|
58819
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
709 |
if (UseCompressedOops) { |
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
710 |
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow)); |
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
711 |
} else { |
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
712 |
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier)); |
ef8be51fff48
8232992: Shenandoah: Implement self-fixing interpreter LRB
zgu
parents:
58737
diff
changeset
|
713 |
} |
57565 | 714 |
__ blr(lr); |
54423 | 715 |
__ mov(rscratch1, r0); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
716 |
__ pop_call_clobbered_registers(); |
54423 | 717 |
__ mov(r0, rscratch1); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
718 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
719 |
__ leave(); // required for proper stackwalking of RuntimeStub frame |
54423 | 720 |
__ bind(done); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
721 |
__ ret(lr); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
722 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
723 |
return start; |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
724 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
725 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
726 |
#undef __ |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
727 |
|
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
728 |
void ShenandoahBarrierSetAssembler::barrier_stubs_init() { |
54423 | 729 |
if (ShenandoahLoadRefBarrier) { |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
730 |
int stub_code_size = 2048; |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
731 |
ResourceMark rm; |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
732 |
BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
733 |
CodeBuffer buf(bb); |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
734 |
StubCodeGenerator cgen(&buf); |
54423 | 735 |
_shenandoah_lrb = generate_shenandoah_lrb(&cgen); |
52925
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
736 |
} |
9c18c9d839d3
8214259: Implementation: JEP 189: Shenandoah: A Low-Pause-Time Garbage Collector (Experimental)
rkennke
parents:
diff
changeset
|
737 |
} |