--- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp Thu Jun 20 14:09:22 2019 +0100
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp Mon Jun 24 11:37:56 2019 -0700
@@ -1433,7 +1433,6 @@
__ jcc(Assembler::greater, L_copy_8_bytes); // Copy trailing qwords
}
-
// Arguments:
// aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
// ignored
@@ -1482,51 +1481,55 @@
setup_arg_regs(); // from => rdi, to => rsi, count => rdx
// r9 and r10 may be used to save non-volatile registers
- // 'from', 'to' and 'count' are now valid
- __ movptr(byte_count, count);
- __ shrptr(count, 3); // count => qword_count
-
- // Copy from low to high addresses. Use 'to' as scratch.
- __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
- __ lea(end_to, Address(to, qword_count, Address::times_8, -8));
- __ negptr(qword_count); // make the count negative
- __ jmp(L_copy_bytes);
-
- // Copy trailing qwords
- __ BIND(L_copy_8_bytes);
- __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
- __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
- __ increment(qword_count);
- __ jcc(Assembler::notZero, L_copy_8_bytes);
-
- // Check for and copy trailing dword
- __ BIND(L_copy_4_bytes);
- __ testl(byte_count, 4);
- __ jccb(Assembler::zero, L_copy_2_bytes);
- __ movl(rax, Address(end_from, 8));
- __ movl(Address(end_to, 8), rax);
-
- __ addptr(end_from, 4);
- __ addptr(end_to, 4);
-
- // Check for and copy trailing word
- __ BIND(L_copy_2_bytes);
- __ testl(byte_count, 2);
- __ jccb(Assembler::zero, L_copy_byte);
- __ movw(rax, Address(end_from, 8));
- __ movw(Address(end_to, 8), rax);
-
- __ addptr(end_from, 2);
- __ addptr(end_to, 2);
-
- // Check for and copy trailing byte
- __ BIND(L_copy_byte);
- __ testl(byte_count, 1);
- __ jccb(Assembler::zero, L_exit);
- __ movb(rax, Address(end_from, 8));
- __ movb(Address(end_to, 8), rax);
-
+ {
+ // UnsafeCopyMemory page error: continue after ucm
+ UnsafeCopyMemoryMark ucmm(this, !aligned, true);
+ // 'from', 'to' and 'count' are now valid
+ __ movptr(byte_count, count);
+ __ shrptr(count, 3); // count => qword_count
+
+ // Copy from low to high addresses. Use 'to' as scratch.
+ __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
+ __ lea(end_to, Address(to, qword_count, Address::times_8, -8));
+ __ negptr(qword_count); // make the count negative
+ __ jmp(L_copy_bytes);
+
+ // Copy trailing qwords
+ __ BIND(L_copy_8_bytes);
+ __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
+ __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
+ __ increment(qword_count);
+ __ jcc(Assembler::notZero, L_copy_8_bytes);
+
+ // Check for and copy trailing dword
+ __ BIND(L_copy_4_bytes);
+ __ testl(byte_count, 4);
+ __ jccb(Assembler::zero, L_copy_2_bytes);
+ __ movl(rax, Address(end_from, 8));
+ __ movl(Address(end_to, 8), rax);
+
+ __ addptr(end_from, 4);
+ __ addptr(end_to, 4);
+
+ // Check for and copy trailing word
+ __ BIND(L_copy_2_bytes);
+ __ testl(byte_count, 2);
+ __ jccb(Assembler::zero, L_copy_byte);
+ __ movw(rax, Address(end_from, 8));
+ __ movw(Address(end_to, 8), rax);
+
+ __ addptr(end_from, 2);
+ __ addptr(end_to, 2);
+
+ // Check for and copy trailing byte
+ __ BIND(L_copy_byte);
+ __ testl(byte_count, 1);
+ __ jccb(Assembler::zero, L_exit);
+ __ movb(rax, Address(end_from, 8));
+ __ movb(Address(end_to, 8), rax);
+ }
__ BIND(L_exit);
+ address ucme_exit_pc = __ pc();
restore_arg_regs();
inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
@@ -1534,10 +1537,12 @@
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
- // Copy in multi-bytes chunks
- copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
- __ jmp(L_copy_4_bytes);
-
+ {
+ UnsafeCopyMemoryMark ucmm(this, !aligned, false, ucme_exit_pc);
+ // Copy in multi-bytes chunks
+ copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+ __ jmp(L_copy_4_bytes);
+ }
return start;
}
@@ -1582,41 +1587,44 @@
setup_arg_regs(); // from => rdi, to => rsi, count => rdx
// r9 and r10 may be used to save non-volatile registers
- // 'from', 'to' and 'count' are now valid
- __ movptr(byte_count, count);
- __ shrptr(count, 3); // count => qword_count
-
- // Copy from high to low addresses.
-
- // Check for and copy trailing byte
- __ testl(byte_count, 1);
- __ jcc(Assembler::zero, L_copy_2_bytes);
- __ movb(rax, Address(from, byte_count, Address::times_1, -1));
- __ movb(Address(to, byte_count, Address::times_1, -1), rax);
- __ decrement(byte_count); // Adjust for possible trailing word
-
- // Check for and copy trailing word
- __ BIND(L_copy_2_bytes);
- __ testl(byte_count, 2);
- __ jcc(Assembler::zero, L_copy_4_bytes);
- __ movw(rax, Address(from, byte_count, Address::times_1, -2));
- __ movw(Address(to, byte_count, Address::times_1, -2), rax);
-
- // Check for and copy trailing dword
- __ BIND(L_copy_4_bytes);
- __ testl(byte_count, 4);
- __ jcc(Assembler::zero, L_copy_bytes);
- __ movl(rax, Address(from, qword_count, Address::times_8));
- __ movl(Address(to, qword_count, Address::times_8), rax);
- __ jmp(L_copy_bytes);
-
- // Copy trailing qwords
- __ BIND(L_copy_8_bytes);
- __ movq(rax, Address(from, qword_count, Address::times_8, -8));
- __ movq(Address(to, qword_count, Address::times_8, -8), rax);
- __ decrement(qword_count);
- __ jcc(Assembler::notZero, L_copy_8_bytes);
-
+ {
+ // UnsafeCopyMemory page error: continue after ucm
+ UnsafeCopyMemoryMark ucmm(this, !aligned, true);
+ // 'from', 'to' and 'count' are now valid
+ __ movptr(byte_count, count);
+ __ shrptr(count, 3); // count => qword_count
+
+ // Copy from high to low addresses.
+
+ // Check for and copy trailing byte
+ __ testl(byte_count, 1);
+ __ jcc(Assembler::zero, L_copy_2_bytes);
+ __ movb(rax, Address(from, byte_count, Address::times_1, -1));
+ __ movb(Address(to, byte_count, Address::times_1, -1), rax);
+ __ decrement(byte_count); // Adjust for possible trailing word
+
+ // Check for and copy trailing word
+ __ BIND(L_copy_2_bytes);
+ __ testl(byte_count, 2);
+ __ jcc(Assembler::zero, L_copy_4_bytes);
+ __ movw(rax, Address(from, byte_count, Address::times_1, -2));
+ __ movw(Address(to, byte_count, Address::times_1, -2), rax);
+
+ // Check for and copy trailing dword
+ __ BIND(L_copy_4_bytes);
+ __ testl(byte_count, 4);
+ __ jcc(Assembler::zero, L_copy_bytes);
+ __ movl(rax, Address(from, qword_count, Address::times_8));
+ __ movl(Address(to, qword_count, Address::times_8), rax);
+ __ jmp(L_copy_bytes);
+
+ // Copy trailing qwords
+ __ BIND(L_copy_8_bytes);
+ __ movq(rax, Address(from, qword_count, Address::times_8, -8));
+ __ movq(Address(to, qword_count, Address::times_8, -8), rax);
+ __ decrement(qword_count);
+ __ jcc(Assembler::notZero, L_copy_8_bytes);
+ }
restore_arg_regs();
inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
@@ -1624,9 +1632,12 @@
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
- // Copy in multi-bytes chunks
- copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
-
+ {
+ // UnsafeCopyMemory page error: continue after ucm
+ UnsafeCopyMemoryMark ucmm(this, !aligned, true);
+ // Copy in multi-bytes chunks
+ copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+ }
restore_arg_regs();
inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
@@ -1684,44 +1695,48 @@
setup_arg_regs(); // from => rdi, to => rsi, count => rdx
// r9 and r10 may be used to save non-volatile registers
- // 'from', 'to' and 'count' are now valid
- __ movptr(word_count, count);
- __ shrptr(count, 2); // count => qword_count
-
- // Copy from low to high addresses. Use 'to' as scratch.
- __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
- __ lea(end_to, Address(to, qword_count, Address::times_8, -8));
- __ negptr(qword_count);
- __ jmp(L_copy_bytes);
-
- // Copy trailing qwords
- __ BIND(L_copy_8_bytes);
- __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
- __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
- __ increment(qword_count);
- __ jcc(Assembler::notZero, L_copy_8_bytes);
-
- // Original 'dest' is trashed, so we can't use it as a
- // base register for a possible trailing word copy
-
- // Check for and copy trailing dword
- __ BIND(L_copy_4_bytes);
- __ testl(word_count, 2);
- __ jccb(Assembler::zero, L_copy_2_bytes);
- __ movl(rax, Address(end_from, 8));
- __ movl(Address(end_to, 8), rax);
-
- __ addptr(end_from, 4);
- __ addptr(end_to, 4);
-
- // Check for and copy trailing word
- __ BIND(L_copy_2_bytes);
- __ testl(word_count, 1);
- __ jccb(Assembler::zero, L_exit);
- __ movw(rax, Address(end_from, 8));
- __ movw(Address(end_to, 8), rax);
-
+ {
+ // UnsafeCopyMemory page error: continue after ucm
+ UnsafeCopyMemoryMark ucmm(this, !aligned, true);
+ // 'from', 'to' and 'count' are now valid
+ __ movptr(word_count, count);
+ __ shrptr(count, 2); // count => qword_count
+
+ // Copy from low to high addresses. Use 'to' as scratch.
+ __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
+ __ lea(end_to, Address(to, qword_count, Address::times_8, -8));
+ __ negptr(qword_count);
+ __ jmp(L_copy_bytes);
+
+ // Copy trailing qwords
+ __ BIND(L_copy_8_bytes);
+ __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
+ __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
+ __ increment(qword_count);
+ __ jcc(Assembler::notZero, L_copy_8_bytes);
+
+ // Original 'dest' is trashed, so we can't use it as a
+ // base register for a possible trailing word copy
+
+ // Check for and copy trailing dword
+ __ BIND(L_copy_4_bytes);
+ __ testl(word_count, 2);
+ __ jccb(Assembler::zero, L_copy_2_bytes);
+ __ movl(rax, Address(end_from, 8));
+ __ movl(Address(end_to, 8), rax);
+
+ __ addptr(end_from, 4);
+ __ addptr(end_to, 4);
+
+ // Check for and copy trailing word
+ __ BIND(L_copy_2_bytes);
+ __ testl(word_count, 1);
+ __ jccb(Assembler::zero, L_exit);
+ __ movw(rax, Address(end_from, 8));
+ __ movw(Address(end_to, 8), rax);
+ }
__ BIND(L_exit);
+ address ucme_exit_pc = __ pc();
restore_arg_regs();
inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
@@ -1729,9 +1744,12 @@
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
- // Copy in multi-bytes chunks
- copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
- __ jmp(L_copy_4_bytes);
+ {
+ UnsafeCopyMemoryMark ucmm(this, !aligned, false, ucme_exit_pc);
+ // Copy in multi-bytes chunks
+ copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+ __ jmp(L_copy_4_bytes);
+ }
return start;
}
@@ -1798,33 +1816,36 @@
setup_arg_regs(); // from => rdi, to => rsi, count => rdx
// r9 and r10 may be used to save non-volatile registers
- // 'from', 'to' and 'count' are now valid
- __ movptr(word_count, count);
- __ shrptr(count, 2); // count => qword_count
-
- // Copy from high to low addresses. Use 'to' as scratch.
-
- // Check for and copy trailing word
- __ testl(word_count, 1);
- __ jccb(Assembler::zero, L_copy_4_bytes);
- __ movw(rax, Address(from, word_count, Address::times_2, -2));
- __ movw(Address(to, word_count, Address::times_2, -2), rax);
-
- // Check for and copy trailing dword
- __ BIND(L_copy_4_bytes);
- __ testl(word_count, 2);
- __ jcc(Assembler::zero, L_copy_bytes);
- __ movl(rax, Address(from, qword_count, Address::times_8));
- __ movl(Address(to, qword_count, Address::times_8), rax);
- __ jmp(L_copy_bytes);
-
- // Copy trailing qwords
- __ BIND(L_copy_8_bytes);
- __ movq(rax, Address(from, qword_count, Address::times_8, -8));
- __ movq(Address(to, qword_count, Address::times_8, -8), rax);
- __ decrement(qword_count);
- __ jcc(Assembler::notZero, L_copy_8_bytes);
-
+ {
+ // UnsafeCopyMemory page error: continue after ucm
+ UnsafeCopyMemoryMark ucmm(this, !aligned, true);
+ // 'from', 'to' and 'count' are now valid
+ __ movptr(word_count, count);
+ __ shrptr(count, 2); // count => qword_count
+
+ // Copy from high to low addresses. Use 'to' as scratch.
+
+ // Check for and copy trailing word
+ __ testl(word_count, 1);
+ __ jccb(Assembler::zero, L_copy_4_bytes);
+ __ movw(rax, Address(from, word_count, Address::times_2, -2));
+ __ movw(Address(to, word_count, Address::times_2, -2), rax);
+
+ // Check for and copy trailing dword
+ __ BIND(L_copy_4_bytes);
+ __ testl(word_count, 2);
+ __ jcc(Assembler::zero, L_copy_bytes);
+ __ movl(rax, Address(from, qword_count, Address::times_8));
+ __ movl(Address(to, qword_count, Address::times_8), rax);
+ __ jmp(L_copy_bytes);
+
+ // Copy trailing qwords
+ __ BIND(L_copy_8_bytes);
+ __ movq(rax, Address(from, qword_count, Address::times_8, -8));
+ __ movq(Address(to, qword_count, Address::times_8, -8), rax);
+ __ decrement(qword_count);
+ __ jcc(Assembler::notZero, L_copy_8_bytes);
+ }
restore_arg_regs();
inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
@@ -1832,9 +1853,12 @@
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
- // Copy in multi-bytes chunks
- copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
-
+ {
+ // UnsafeCopyMemory page error: continue after ucm
+ UnsafeCopyMemoryMark ucmm(this, !aligned, true);
+ // Copy in multi-bytes chunks
+ copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+ }
restore_arg_regs();
inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
@@ -1905,31 +1929,35 @@
BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
- // 'from', 'to' and 'count' are now valid
- __ movptr(dword_count, count);
- __ shrptr(count, 1); // count => qword_count
-
- // Copy from low to high addresses. Use 'to' as scratch.
- __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
- __ lea(end_to, Address(to, qword_count, Address::times_8, -8));
- __ negptr(qword_count);
- __ jmp(L_copy_bytes);
-
- // Copy trailing qwords
- __ BIND(L_copy_8_bytes);
- __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
- __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
- __ increment(qword_count);
- __ jcc(Assembler::notZero, L_copy_8_bytes);
-
- // Check for and copy trailing dword
- __ BIND(L_copy_4_bytes);
- __ testl(dword_count, 1); // Only byte test since the value is 0 or 1
- __ jccb(Assembler::zero, L_exit);
- __ movl(rax, Address(end_from, 8));
- __ movl(Address(end_to, 8), rax);
-
+ {
+ // UnsafeCopyMemory page error: continue after ucm
+ UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
+ // 'from', 'to' and 'count' are now valid
+ __ movptr(dword_count, count);
+ __ shrptr(count, 1); // count => qword_count
+
+ // Copy from low to high addresses. Use 'to' as scratch.
+ __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
+ __ lea(end_to, Address(to, qword_count, Address::times_8, -8));
+ __ negptr(qword_count);
+ __ jmp(L_copy_bytes);
+
+ // Copy trailing qwords
+ __ BIND(L_copy_8_bytes);
+ __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
+ __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
+ __ increment(qword_count);
+ __ jcc(Assembler::notZero, L_copy_8_bytes);
+
+ // Check for and copy trailing dword
+ __ BIND(L_copy_4_bytes);
+ __ testl(dword_count, 1); // Only byte test since the value is 0 or 1
+ __ jccb(Assembler::zero, L_exit);
+ __ movl(rax, Address(end_from, 8));
+ __ movl(Address(end_to, 8), rax);
+ }
__ BIND(L_exit);
+ address ucme_exit_pc = __ pc();
bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
restore_arg_regs_using_thread();
inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
@@ -1938,9 +1966,12 @@
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
- // Copy in multi-bytes chunks
- copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
- __ jmp(L_copy_4_bytes);
+ {
+ UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, false, ucme_exit_pc);
+ // Copy in multi-bytes chunks
+ copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+ __ jmp(L_copy_4_bytes);
+ }
return start;
}
@@ -2001,26 +2032,29 @@
bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
assert_clean_int(count, rax); // Make sure 'count' is clean int.
- // 'from', 'to' and 'count' are now valid
- __ movptr(dword_count, count);
- __ shrptr(count, 1); // count => qword_count
-
- // Copy from high to low addresses. Use 'to' as scratch.
-
- // Check for and copy trailing dword
- __ testl(dword_count, 1);
- __ jcc(Assembler::zero, L_copy_bytes);
- __ movl(rax, Address(from, dword_count, Address::times_4, -4));
- __ movl(Address(to, dword_count, Address::times_4, -4), rax);
- __ jmp(L_copy_bytes);
-
- // Copy trailing qwords
- __ BIND(L_copy_8_bytes);
- __ movq(rax, Address(from, qword_count, Address::times_8, -8));
- __ movq(Address(to, qword_count, Address::times_8, -8), rax);
- __ decrement(qword_count);
- __ jcc(Assembler::notZero, L_copy_8_bytes);
-
+ {
+ // UnsafeCopyMemory page error: continue after ucm
+ UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
+ // 'from', 'to' and 'count' are now valid
+ __ movptr(dword_count, count);
+ __ shrptr(count, 1); // count => qword_count
+
+ // Copy from high to low addresses. Use 'to' as scratch.
+
+ // Check for and copy trailing dword
+ __ testl(dword_count, 1);
+ __ jcc(Assembler::zero, L_copy_bytes);
+ __ movl(rax, Address(from, dword_count, Address::times_4, -4));
+ __ movl(Address(to, dword_count, Address::times_4, -4), rax);
+ __ jmp(L_copy_bytes);
+
+ // Copy trailing qwords
+ __ BIND(L_copy_8_bytes);
+ __ movq(rax, Address(from, qword_count, Address::times_8, -8));
+ __ movq(Address(to, qword_count, Address::times_8, -8), rax);
+ __ decrement(qword_count);
+ __ jcc(Assembler::notZero, L_copy_8_bytes);
+ }
if (is_oop) {
__ jmp(L_exit);
}
@@ -2031,8 +2065,12 @@
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
- // Copy in multi-bytes chunks
- copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+ {
+ // UnsafeCopyMemory page error: continue after ucm
+ UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
+ // Copy in multi-bytes chunks
+ copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+ }
__ BIND(L_exit);
bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
@@ -2102,20 +2140,23 @@
BasicType type = is_oop ? T_OBJECT : T_LONG;
BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
-
- // Copy from low to high addresses. Use 'to' as scratch.
- __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
- __ lea(end_to, Address(to, qword_count, Address::times_8, -8));
- __ negptr(qword_count);
- __ jmp(L_copy_bytes);
-
- // Copy trailing qwords
- __ BIND(L_copy_8_bytes);
- __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
- __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
- __ increment(qword_count);
- __ jcc(Assembler::notZero, L_copy_8_bytes);
-
+ {
+ // UnsafeCopyMemory page error: continue after ucm
+ UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
+
+ // Copy from low to high addresses. Use 'to' as scratch.
+ __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
+ __ lea(end_to, Address(to, qword_count, Address::times_8, -8));
+ __ negptr(qword_count);
+ __ jmp(L_copy_bytes);
+
+ // Copy trailing qwords
+ __ BIND(L_copy_8_bytes);
+ __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
+ __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
+ __ increment(qword_count);
+ __ jcc(Assembler::notZero, L_copy_8_bytes);
+ }
if (is_oop) {
__ jmp(L_exit);
} else {
@@ -2127,8 +2168,12 @@
__ ret(0);
}
- // Copy in multi-bytes chunks
- copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+ {
+ // UnsafeCopyMemory page error: continue after ucm
+ UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
+ // Copy in multi-bytes chunks
+ copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+ }
__ BIND(L_exit);
bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count);
@@ -2195,16 +2240,19 @@
BasicType type = is_oop ? T_OBJECT : T_LONG;
BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
-
- __ jmp(L_copy_bytes);
-
- // Copy trailing qwords
- __ BIND(L_copy_8_bytes);
- __ movq(rax, Address(from, qword_count, Address::times_8, -8));
- __ movq(Address(to, qword_count, Address::times_8, -8), rax);
- __ decrement(qword_count);
- __ jcc(Assembler::notZero, L_copy_8_bytes);
-
+ {
+ // UnsafeCopyMemory page error: continue after ucm
+ UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
+
+ __ jmp(L_copy_bytes);
+
+ // Copy trailing qwords
+ __ BIND(L_copy_8_bytes);
+ __ movq(rax, Address(from, qword_count, Address::times_8, -8));
+ __ movq(Address(to, qword_count, Address::times_8, -8), rax);
+ __ decrement(qword_count);
+ __ jcc(Assembler::notZero, L_copy_8_bytes);
+ }
if (is_oop) {
__ jmp(L_exit);
} else {
@@ -2215,10 +2263,13 @@
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
}
-
- // Copy in multi-bytes chunks
- copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
-
+ {
+ // UnsafeCopyMemory page error: continue after ucm
+ UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
+
+ // Copy in multi-bytes chunks
+ copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+ }
__ BIND(L_exit);
bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count);
restore_arg_regs_using_thread();
@@ -6036,6 +6087,10 @@
}
}; // end class declaration
+#define UCM_TABLE_MAX_ENTRIES 16
void StubGenerator_generate(CodeBuffer* code, bool all) {
+ if (UnsafeCopyMemory::_table == NULL) {
+ UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES);
+ }
StubGenerator g(code, all);
}