jdk-sandbox: comparison src/hotspot/cpu/x86/stubGenerator_x86

equal deleted inserted replaced

-:13588c901957
+:9cf78a70fa4f
 __ mov(r12, rsp);                               // remember rsp
 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
 __ andptr(rsp, -16);                            // align stack as required by ABI
 BLOCK_COMMENT("call MacroAssembler::debug");
 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
-__ mov(rsp, r12);                               // restore rsp
+__ hlt();
-__ popa();                                      // pop registers (includes r12)
-__ ret(4 * wordSize);                           // pop caller saved stuff
 return start;
 }
 //
 // Verify that a register contains clean 32-bits positive value
 Label L_loop;
 __ align(OptoLoopAlignment);
 if (UseUnalignedLoadStores) {
 Label L_end;
 // Copy 64-bytes per iteration
-__ BIND(L_loop);
 if (UseAVX > 2) {
+Label L_loop_avx512, L_loop_avx2, L_32_byte_head, L_above_threshold, L_below_threshold;
+__ BIND(L_copy_bytes);
+__ cmpptr(qword_count, (-1 * AVX3Threshold / 8));
+__ jccb(Assembler::less, L_above_threshold);
+__ jmpb(L_below_threshold);
+__ bind(L_loop_avx512);
 __ evmovdqul(xmm0, Address(end_from, qword_count, Address::times_8, -56), Assembler::AVX_512bit);
 __ evmovdqul(Address(end_to, qword_count, Address::times_8, -56), xmm0, Assembler::AVX_512bit);
-} else if (UseAVX == 2) {
+__ bind(L_above_threshold);
+__ addptr(qword_count, 8);
+__ jcc(Assembler::lessEqual, L_loop_avx512);
+__ jmpb(L_32_byte_head);
+__ bind(L_loop_avx2);
 __ vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
 __ vmovdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
 __ vmovdqu(xmm1, Address(end_from, qword_count, Address::times_8, -24));
 __ vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm1);
+__ bind(L_below_threshold);
+__ addptr(qword_count, 8);
+__ jcc(Assembler::lessEqual, L_loop_avx2);
+__ bind(L_32_byte_head);
+__ subptr(qword_count, 4);  // sub(8) and add(4)
+__ jccb(Assembler::greater, L_end);
 } else {
-__ movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
+__ BIND(L_loop);
-__ movdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
+if (UseAVX == 2) {
-__ movdqu(xmm1, Address(end_from, qword_count, Address::times_8, -40));
+__ vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
-__ movdqu(Address(end_to, qword_count, Address::times_8, -40), xmm1);
+__ vmovdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
-__ movdqu(xmm2, Address(end_from, qword_count, Address::times_8, -24));
+__ vmovdqu(xmm1, Address(end_from, qword_count, Address::times_8, -24));
-__ movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm2);
+__ vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm1);
-__ movdqu(xmm3, Address(end_from, qword_count, Address::times_8, - 8));
+} else {
-__ movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm3);
+__ movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
+__ movdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
+__ movdqu(xmm1, Address(end_from, qword_count, Address::times_8, -40));
+__ movdqu(Address(end_to, qword_count, Address::times_8, -40), xmm1);
+__ movdqu(xmm2, Address(end_from, qword_count, Address::times_8, -24));
+__ movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm2);
+__ movdqu(xmm3, Address(end_from, qword_count, Address::times_8, - 8));
+__ movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm3);
+}
+__ BIND(L_copy_bytes);
+__ addptr(qword_count, 8);
+__ jcc(Assembler::lessEqual, L_loop);
+__ subptr(qword_count, 4);  // sub(8) and add(4)
+__ jccb(Assembler::greater, L_end);
 }
-__ BIND(L_copy_bytes);
-__ addptr(qword_count, 8);
-__ jcc(Assembler::lessEqual, L_loop);
-__ subptr(qword_count, 4);  // sub(8) and add(4)
-__ jccb(Assembler::greater, L_end);
 // Copy trailing 32 bytes
 if (UseAVX >= 2) {
 __ vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24));
 __ vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0);
 } else {
 Label L_loop;
 __ align(OptoLoopAlignment);
 if (UseUnalignedLoadStores) {
 Label L_end;
 // Copy 64-bytes per iteration
-__ BIND(L_loop);
 if (UseAVX > 2) {
+Label L_loop_avx512, L_loop_avx2, L_32_byte_head, L_above_threshold, L_below_threshold;
+__ BIND(L_copy_bytes);
+__ cmpptr(qword_count, (AVX3Threshold / 8));
+__ jccb(Assembler::greater, L_above_threshold);
+__ jmpb(L_below_threshold);
+__ BIND(L_loop_avx512);
 __ evmovdqul(xmm0, Address(from, qword_count, Address::times_8, 0), Assembler::AVX_512bit);
 __ evmovdqul(Address(dest, qword_count, Address::times_8, 0), xmm0, Assembler::AVX_512bit);
-} else if (UseAVX == 2) {
+__ bind(L_above_threshold);
+__ subptr(qword_count, 8);
+__ jcc(Assembler::greaterEqual, L_loop_avx512);
+__ jmpb(L_32_byte_head);
+__ bind(L_loop_avx2);
 __ vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 32));
 __ vmovdqu(Address(dest, qword_count, Address::times_8, 32), xmm0);
-__ vmovdqu(xmm1, Address(from, qword_count, Address::times_8,  0));
+__ vmovdqu(xmm1, Address(from, qword_count, Address::times_8, 0));
-__ vmovdqu(Address(dest, qword_count, Address::times_8,  0), xmm1);
+__ vmovdqu(Address(dest, qword_count, Address::times_8, 0), xmm1);
+__ bind(L_below_threshold);
+__ subptr(qword_count, 8);
+__ jcc(Assembler::greaterEqual, L_loop_avx2);
+__ bind(L_32_byte_head);
+__ addptr(qword_count, 4);  // add(8) and sub(4)
+__ jccb(Assembler::less, L_end);
 } else {
-__ movdqu(xmm0, Address(from, qword_count, Address::times_8, 48));
+__ BIND(L_loop);
-__ movdqu(Address(dest, qword_count, Address::times_8, 48), xmm0);
+if (UseAVX == 2) {
-__ movdqu(xmm1, Address(from, qword_count, Address::times_8, 32));
+__ vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 32));
-__ movdqu(Address(dest, qword_count, Address::times_8, 32), xmm1);
+__ vmovdqu(Address(dest, qword_count, Address::times_8, 32), xmm0);
-__ movdqu(xmm2, Address(from, qword_count, Address::times_8, 16));
+__ vmovdqu(xmm1, Address(from, qword_count, Address::times_8,  0));
-__ movdqu(Address(dest, qword_count, Address::times_8, 16), xmm2);
+__ vmovdqu(Address(dest, qword_count, Address::times_8,  0), xmm1);
-__ movdqu(xmm3, Address(from, qword_count, Address::times_8,  0));
+} else {
-__ movdqu(Address(dest, qword_count, Address::times_8,  0), xmm3);
+__ movdqu(xmm0, Address(from, qword_count, Address::times_8, 48));
+__ movdqu(Address(dest, qword_count, Address::times_8, 48), xmm0);
+__ movdqu(xmm1, Address(from, qword_count, Address::times_8, 32));
+__ movdqu(Address(dest, qword_count, Address::times_8, 32), xmm1);
+__ movdqu(xmm2, Address(from, qword_count, Address::times_8, 16));
+__ movdqu(Address(dest, qword_count, Address::times_8, 16), xmm2);
+__ movdqu(xmm3, Address(from, qword_count, Address::times_8,  0));
+__ movdqu(Address(dest, qword_count, Address::times_8,  0), xmm3);
+}
+__ BIND(L_copy_bytes);
+__ subptr(qword_count, 8);
+__ jcc(Assembler::greaterEqual, L_loop);
+__ addptr(qword_count, 4);  // add(8) and sub(4)
+__ jccb(Assembler::less, L_end);
 }
-__ BIND(L_copy_bytes);
-__ subptr(qword_count, 8);
-__ jcc(Assembler::greaterEqual, L_loop);
-__ addptr(qword_count, 4);  // add(8) and sub(4)
-__ jccb(Assembler::less, L_end);
 // Copy trailing 32 bytes
 if (UseAVX >= 2) {
 __ vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 0));
 __ vmovdqu(Address(dest, qword_count, Address::times_8, 0), xmm0);
 } else {
 }
 __ addptr(qword_count, 4);
 __ jcc(Assembler::greater, L_copy_8_bytes); // Copy trailing qwords
 }
 // Arguments:
 //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
 //             ignored
 //   name    - stub name string
 //
 }
 setup_arg_regs(); // from => rdi, to => rsi, count => rdx
 // r9 and r10 may be used to save non-volatile registers
-// 'from', 'to' and 'count' are now valid
+{
-__ movptr(byte_count, count);
+// UnsafeCopyMemory page error: continue after ucm
-__ shrptr(count, 3); // count => qword_count
+UnsafeCopyMemoryMark ucmm(this, !aligned, true);
+// 'from', 'to' and 'count' are now valid
-// Copy from low to high addresses.  Use 'to' as scratch.
+__ movptr(byte_count, count);
-__ lea(end_from, Address(from, qword_count, Address::times_8, -8));
+__ shrptr(count, 3); // count => qword_count
-__ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
-__ negptr(qword_count); // make the count negative
+// Copy from low to high addresses.  Use 'to' as scratch.
-__ jmp(L_copy_bytes);
+__ lea(end_from, Address(from, qword_count, Address::times_8, -8));
+__ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
-// Copy trailing qwords
+__ negptr(qword_count); // make the count negative
-__ BIND(L_copy_8_bytes);
+__ jmp(L_copy_bytes);
-__ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
-__ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
+// Copy trailing qwords
-__ increment(qword_count);
+__ BIND(L_copy_8_bytes);
-__ jcc(Assembler::notZero, L_copy_8_bytes);
+__ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
+__ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
-// Check for and copy trailing dword
+__ increment(qword_count);
-__ BIND(L_copy_4_bytes);
+__ jcc(Assembler::notZero, L_copy_8_bytes);
-__ testl(byte_count, 4);
-__ jccb(Assembler::zero, L_copy_2_bytes);
+// Check for and copy trailing dword
-__ movl(rax, Address(end_from, 8));
+__ BIND(L_copy_4_bytes);
-__ movl(Address(end_to, 8), rax);
+__ testl(byte_count, 4);
+__ jccb(Assembler::zero, L_copy_2_bytes);
-__ addptr(end_from, 4);
+__ movl(rax, Address(end_from, 8));
-__ addptr(end_to, 4);
+__ movl(Address(end_to, 8), rax);
-// Check for and copy trailing word
+__ addptr(end_from, 4);
-__ BIND(L_copy_2_bytes);
+__ addptr(end_to, 4);
-__ testl(byte_count, 2);
-__ jccb(Assembler::zero, L_copy_byte);
+// Check for and copy trailing word
-__ movw(rax, Address(end_from, 8));
+__ BIND(L_copy_2_bytes);
-__ movw(Address(end_to, 8), rax);
+__ testl(byte_count, 2);
+__ jccb(Assembler::zero, L_copy_byte);
-__ addptr(end_from, 2);
+__ movw(rax, Address(end_from, 8));
-__ addptr(end_to, 2);
+__ movw(Address(end_to, 8), rax);
-// Check for and copy trailing byte
+__ addptr(end_from, 2);
-__ BIND(L_copy_byte);
+__ addptr(end_to, 2);
-__ testl(byte_count, 1);
-__ jccb(Assembler::zero, L_exit);
+// Check for and copy trailing byte
-__ movb(rax, Address(end_from, 8));
+__ BIND(L_copy_byte);
-__ movb(Address(end_to, 8), rax);
+__ testl(byte_count, 1);
+__ jccb(Assembler::zero, L_exit);
+__ movb(rax, Address(end_from, 8));
+__ movb(Address(end_to, 8), rax);
+}
 __ BIND(L_exit);
+address ucme_exit_pc = __ pc();
 restore_arg_regs();
 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
 __ xorptr(rax, rax); // return 0
 __ vzeroupper();
 __ leave(); // required for proper stackwalking of RuntimeStub frame
 __ ret(0);
-// Copy in multi-bytes chunks
+{
-copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+UnsafeCopyMemoryMark ucmm(this, !aligned, false, ucme_exit_pc);
-__ jmp(L_copy_4_bytes);
+// Copy in multi-bytes chunks
+copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+__ jmp(L_copy_4_bytes);
+}
 return start;
 }
 // Arguments:
 //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
 array_overlap_test(nooverlap_target, Address::times_1);
 setup_arg_regs(); // from => rdi, to => rsi, count => rdx
 // r9 and r10 may be used to save non-volatile registers
-// 'from', 'to' and 'count' are now valid
+{
-__ movptr(byte_count, count);
+// UnsafeCopyMemory page error: continue after ucm
-__ shrptr(count, 3);   // count => qword_count
+UnsafeCopyMemoryMark ucmm(this, !aligned, true);
+// 'from', 'to' and 'count' are now valid
-// Copy from high to low addresses.
+__ movptr(byte_count, count);
+__ shrptr(count, 3);   // count => qword_count
-// Check for and copy trailing byte
-__ testl(byte_count, 1);
+// Copy from high to low addresses.
-__ jcc(Assembler::zero, L_copy_2_bytes);
-__ movb(rax, Address(from, byte_count, Address::times_1, -1));
+// Check for and copy trailing byte
-__ movb(Address(to, byte_count, Address::times_1, -1), rax);
+__ testl(byte_count, 1);
-__ decrement(byte_count); // Adjust for possible trailing word
+__ jcc(Assembler::zero, L_copy_2_bytes);
+__ movb(rax, Address(from, byte_count, Address::times_1, -1));
-// Check for and copy trailing word
+__ movb(Address(to, byte_count, Address::times_1, -1), rax);
-__ BIND(L_copy_2_bytes);
+__ decrement(byte_count); // Adjust for possible trailing word
-__ testl(byte_count, 2);
-__ jcc(Assembler::zero, L_copy_4_bytes);
+// Check for and copy trailing word
-__ movw(rax, Address(from, byte_count, Address::times_1, -2));
+__ BIND(L_copy_2_bytes);
-__ movw(Address(to, byte_count, Address::times_1, -2), rax);
+__ testl(byte_count, 2);
+__ jcc(Assembler::zero, L_copy_4_bytes);
-// Check for and copy trailing dword
+__ movw(rax, Address(from, byte_count, Address::times_1, -2));
-__ BIND(L_copy_4_bytes);
+__ movw(Address(to, byte_count, Address::times_1, -2), rax);
-__ testl(byte_count, 4);
-__ jcc(Assembler::zero, L_copy_bytes);
+// Check for and copy trailing dword
-__ movl(rax, Address(from, qword_count, Address::times_8));
+__ BIND(L_copy_4_bytes);
-__ movl(Address(to, qword_count, Address::times_8), rax);
+__ testl(byte_count, 4);
-__ jmp(L_copy_bytes);
+__ jcc(Assembler::zero, L_copy_bytes);
+__ movl(rax, Address(from, qword_count, Address::times_8));
-// Copy trailing qwords
+__ movl(Address(to, qword_count, Address::times_8), rax);
-__ BIND(L_copy_8_bytes);
+__ jmp(L_copy_bytes);
-__ movq(rax, Address(from, qword_count, Address::times_8, -8));
-__ movq(Address(to, qword_count, Address::times_8, -8), rax);
+// Copy trailing qwords
-__ decrement(qword_count);
+__ BIND(L_copy_8_bytes);
-__ jcc(Assembler::notZero, L_copy_8_bytes);
+__ movq(rax, Address(from, qword_count, Address::times_8, -8));
+__ movq(Address(to, qword_count, Address::times_8, -8), rax);
+__ decrement(qword_count);
+__ jcc(Assembler::notZero, L_copy_8_bytes);
+}
 restore_arg_regs();
 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
 __ xorptr(rax, rax); // return 0
 __ vzeroupper();
 __ leave(); // required for proper stackwalking of RuntimeStub frame
 __ ret(0);
-// Copy in multi-bytes chunks
+{
-copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+// UnsafeCopyMemory page error: continue after ucm
+UnsafeCopyMemoryMark ucmm(this, !aligned, true);
+// Copy in multi-bytes chunks
+copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+}
 restore_arg_regs();
 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
 __ xorptr(rax, rax); // return 0
 __ vzeroupper();
 __ leave(); // required for proper stackwalking of RuntimeStub frame
 }
 setup_arg_regs(); // from => rdi, to => rsi, count => rdx
 // r9 and r10 may be used to save non-volatile registers
-// 'from', 'to' and 'count' are now valid
+{
-__ movptr(word_count, count);
+// UnsafeCopyMemory page error: continue after ucm
-__ shrptr(count, 2); // count => qword_count
+UnsafeCopyMemoryMark ucmm(this, !aligned, true);
+// 'from', 'to' and 'count' are now valid
-// Copy from low to high addresses.  Use 'to' as scratch.
+__ movptr(word_count, count);
-__ lea(end_from, Address(from, qword_count, Address::times_8, -8));
+__ shrptr(count, 2); // count => qword_count
-__ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
-__ negptr(qword_count);
+// Copy from low to high addresses.  Use 'to' as scratch.
-__ jmp(L_copy_bytes);
+__ lea(end_from, Address(from, qword_count, Address::times_8, -8));
+__ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
-// Copy trailing qwords
+__ negptr(qword_count);
-__ BIND(L_copy_8_bytes);
+__ jmp(L_copy_bytes);
-__ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
-__ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
+// Copy trailing qwords
-__ increment(qword_count);
+__ BIND(L_copy_8_bytes);
-__ jcc(Assembler::notZero, L_copy_8_bytes);
+__ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
+__ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
-// Original 'dest' is trashed, so we can't use it as a
+__ increment(qword_count);
-// base register for a possible trailing word copy
+__ jcc(Assembler::notZero, L_copy_8_bytes);
-// Check for and copy trailing dword
+// Original 'dest' is trashed, so we can't use it as a
-__ BIND(L_copy_4_bytes);
+// base register for a possible trailing word copy
-__ testl(word_count, 2);
-__ jccb(Assembler::zero, L_copy_2_bytes);
+// Check for and copy trailing dword
-__ movl(rax, Address(end_from, 8));
+__ BIND(L_copy_4_bytes);
-__ movl(Address(end_to, 8), rax);
+__ testl(word_count, 2);
+__ jccb(Assembler::zero, L_copy_2_bytes);
-__ addptr(end_from, 4);
+__ movl(rax, Address(end_from, 8));
-__ addptr(end_to, 4);
+__ movl(Address(end_to, 8), rax);
-// Check for and copy trailing word
+__ addptr(end_from, 4);
-__ BIND(L_copy_2_bytes);
+__ addptr(end_to, 4);
-__ testl(word_count, 1);
-__ jccb(Assembler::zero, L_exit);
+// Check for and copy trailing word
-__ movw(rax, Address(end_from, 8));
+__ BIND(L_copy_2_bytes);
-__ movw(Address(end_to, 8), rax);
+__ testl(word_count, 1);
+__ jccb(Assembler::zero, L_exit);
+__ movw(rax, Address(end_from, 8));
+__ movw(Address(end_to, 8), rax);
+}
 __ BIND(L_exit);
+address ucme_exit_pc = __ pc();
 restore_arg_regs();
 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
 __ xorptr(rax, rax); // return 0
 __ vzeroupper();
 __ leave(); // required for proper stackwalking of RuntimeStub frame
 __ ret(0);
-// Copy in multi-bytes chunks
+{
-copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+UnsafeCopyMemoryMark ucmm(this, !aligned, false, ucme_exit_pc);
-__ jmp(L_copy_4_bytes);
+// Copy in multi-bytes chunks
+copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+__ jmp(L_copy_4_bytes);
+}
 return start;
 }
 address generate_fill(BasicType t, bool aligned, const char *name) {
 array_overlap_test(nooverlap_target, Address::times_2);
 setup_arg_regs(); // from => rdi, to => rsi, count => rdx
 // r9 and r10 may be used to save non-volatile registers
-// 'from', 'to' and 'count' are now valid
+{
-__ movptr(word_count, count);
+// UnsafeCopyMemory page error: continue after ucm
-__ shrptr(count, 2); // count => qword_count
+UnsafeCopyMemoryMark ucmm(this, !aligned, true);
+// 'from', 'to' and 'count' are now valid
-// Copy from high to low addresses.  Use 'to' as scratch.
+__ movptr(word_count, count);
+__ shrptr(count, 2); // count => qword_count
-// Check for and copy trailing word
-__ testl(word_count, 1);
+// Copy from high to low addresses.  Use 'to' as scratch.
-__ jccb(Assembler::zero, L_copy_4_bytes);
-__ movw(rax, Address(from, word_count, Address::times_2, -2));
+// Check for and copy trailing word
-__ movw(Address(to, word_count, Address::times_2, -2), rax);
+__ testl(word_count, 1);
+__ jccb(Assembler::zero, L_copy_4_bytes);
-// Check for and copy trailing dword
+__ movw(rax, Address(from, word_count, Address::times_2, -2));
-__ BIND(L_copy_4_bytes);
+__ movw(Address(to, word_count, Address::times_2, -2), rax);
-__ testl(word_count, 2);
-__ jcc(Assembler::zero, L_copy_bytes);
+// Check for and copy trailing dword
-__ movl(rax, Address(from, qword_count, Address::times_8));
+__ BIND(L_copy_4_bytes);
-__ movl(Address(to, qword_count, Address::times_8), rax);
+__ testl(word_count, 2);
-__ jmp(L_copy_bytes);
+__ jcc(Assembler::zero, L_copy_bytes);
+__ movl(rax, Address(from, qword_count, Address::times_8));
-// Copy trailing qwords
+__ movl(Address(to, qword_count, Address::times_8), rax);
-__ BIND(L_copy_8_bytes);
+__ jmp(L_copy_bytes);
-__ movq(rax, Address(from, qword_count, Address::times_8, -8));
-__ movq(Address(to, qword_count, Address::times_8, -8), rax);
+// Copy trailing qwords
-__ decrement(qword_count);
+__ BIND(L_copy_8_bytes);
-__ jcc(Assembler::notZero, L_copy_8_bytes);
+__ movq(rax, Address(from, qword_count, Address::times_8, -8));
+__ movq(Address(to, qword_count, Address::times_8, -8), rax);
+__ decrement(qword_count);
+__ jcc(Assembler::notZero, L_copy_8_bytes);
+}
 restore_arg_regs();
 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
 __ xorptr(rax, rax); // return 0
 __ vzeroupper();
 __ leave(); // required for proper stackwalking of RuntimeStub frame
 __ ret(0);
-// Copy in multi-bytes chunks
+{
-copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+// UnsafeCopyMemory page error: continue after ucm
+UnsafeCopyMemoryMark ucmm(this, !aligned, true);
+// Copy in multi-bytes chunks
+copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+}
 restore_arg_regs();
 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
 __ xorptr(rax, rax); // return 0
 __ vzeroupper();
 __ leave(); // required for proper stackwalking of RuntimeStub frame
 BasicType type = is_oop ? T_OBJECT : T_INT;
 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
 bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
-// 'from', 'to' and 'count' are now valid
+{
-__ movptr(dword_count, count);
+// UnsafeCopyMemory page error: continue after ucm
-__ shrptr(count, 1); // count => qword_count
+UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
+// 'from', 'to' and 'count' are now valid
-// Copy from low to high addresses.  Use 'to' as scratch.
+__ movptr(dword_count, count);
-__ lea(end_from, Address(from, qword_count, Address::times_8, -8));
+__ shrptr(count, 1); // count => qword_count
-__ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
-__ negptr(qword_count);
+// Copy from low to high addresses.  Use 'to' as scratch.
-__ jmp(L_copy_bytes);
+__ lea(end_from, Address(from, qword_count, Address::times_8, -8));
+__ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
-// Copy trailing qwords
+__ negptr(qword_count);
-__ BIND(L_copy_8_bytes);
+__ jmp(L_copy_bytes);
-__ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
-__ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
+// Copy trailing qwords
-__ increment(qword_count);
+__ BIND(L_copy_8_bytes);
-__ jcc(Assembler::notZero, L_copy_8_bytes);
+__ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
+__ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
-// Check for and copy trailing dword
+__ increment(qword_count);
-__ BIND(L_copy_4_bytes);
+__ jcc(Assembler::notZero, L_copy_8_bytes);
-__ testl(dword_count, 1); // Only byte test since the value is 0 or 1
-__ jccb(Assembler::zero, L_exit);
+// Check for and copy trailing dword
-__ movl(rax, Address(end_from, 8));
+__ BIND(L_copy_4_bytes);
-__ movl(Address(end_to, 8), rax);
+__ testl(dword_count, 1); // Only byte test since the value is 0 or 1
+__ jccb(Assembler::zero, L_exit);
+__ movl(rax, Address(end_from, 8));
+__ movl(Address(end_to, 8), rax);
+}
 __ BIND(L_exit);
+address ucme_exit_pc = __ pc();
 bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
 restore_arg_regs_using_thread();
 inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
 __ vzeroupper();
 __ xorptr(rax, rax); // return 0
 __ leave(); // required for proper stackwalking of RuntimeStub frame
 __ ret(0);
-// Copy in multi-bytes chunks
+{
-copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, false, ucme_exit_pc);
-__ jmp(L_copy_4_bytes);
+// Copy in multi-bytes chunks
+copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+__ jmp(L_copy_4_bytes);
+}
 return start;
 }
 // Arguments:
 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
 // no registers are destroyed by this call
 bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
 assert_clean_int(count, rax); // Make sure 'count' is clean int.
-// 'from', 'to' and 'count' are now valid
+{
-__ movptr(dword_count, count);
+// UnsafeCopyMemory page error: continue after ucm
-__ shrptr(count, 1); // count => qword_count
+UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
+// 'from', 'to' and 'count' are now valid
-// Copy from high to low addresses.  Use 'to' as scratch.
+__ movptr(dword_count, count);
+__ shrptr(count, 1); // count => qword_count
-// Check for and copy trailing dword
-__ testl(dword_count, 1);
+// Copy from high to low addresses.  Use 'to' as scratch.
-__ jcc(Assembler::zero, L_copy_bytes);
-__ movl(rax, Address(from, dword_count, Address::times_4, -4));
+// Check for and copy trailing dword
-__ movl(Address(to, dword_count, Address::times_4, -4), rax);
+__ testl(dword_count, 1);
-__ jmp(L_copy_bytes);
+__ jcc(Assembler::zero, L_copy_bytes);
+__ movl(rax, Address(from, dword_count, Address::times_4, -4));
-// Copy trailing qwords
+__ movl(Address(to, dword_count, Address::times_4, -4), rax);
-__ BIND(L_copy_8_bytes);
+__ jmp(L_copy_bytes);
-__ movq(rax, Address(from, qword_count, Address::times_8, -8));
-__ movq(Address(to, qword_count, Address::times_8, -8), rax);
+// Copy trailing qwords
-__ decrement(qword_count);
+__ BIND(L_copy_8_bytes);
-__ jcc(Assembler::notZero, L_copy_8_bytes);
+__ movq(rax, Address(from, qword_count, Address::times_8, -8));
+__ movq(Address(to, qword_count, Address::times_8, -8), rax);
+__ decrement(qword_count);
+__ jcc(Assembler::notZero, L_copy_8_bytes);
+}
 if (is_oop) {
 __ jmp(L_exit);
 }
 restore_arg_regs_using_thread();
 inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
 __ xorptr(rax, rax); // return 0
 __ vzeroupper();
 __ leave(); // required for proper stackwalking of RuntimeStub frame
 __ ret(0);
-// Copy in multi-bytes chunks
+{
-copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+// UnsafeCopyMemory page error: continue after ucm
+UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
+// Copy in multi-bytes chunks
+copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+}
 __ BIND(L_exit);
 bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
 restore_arg_regs_using_thread();
 inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
 }
 BasicType type = is_oop ? T_OBJECT : T_LONG;
 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
 bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
+{
-// Copy from low to high addresses.  Use 'to' as scratch.
+// UnsafeCopyMemory page error: continue after ucm
-__ lea(end_from, Address(from, qword_count, Address::times_8, -8));
+UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
-__ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
-__ negptr(qword_count);
+// Copy from low to high addresses.  Use 'to' as scratch.
-__ jmp(L_copy_bytes);
+__ lea(end_from, Address(from, qword_count, Address::times_8, -8));
+__ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
-// Copy trailing qwords
+__ negptr(qword_count);
-__ BIND(L_copy_8_bytes);
+__ jmp(L_copy_bytes);
-__ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
-__ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
+// Copy trailing qwords
-__ increment(qword_count);
+__ BIND(L_copy_8_bytes);
-__ jcc(Assembler::notZero, L_copy_8_bytes);
+__ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
+__ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
+__ increment(qword_count);
+__ jcc(Assembler::notZero, L_copy_8_bytes);
+}
 if (is_oop) {
 __ jmp(L_exit);
 } else {
 restore_arg_regs_using_thread();
 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
 __ vzeroupper();
 __ leave(); // required for proper stackwalking of RuntimeStub frame
 __ ret(0);
 }
-// Copy in multi-bytes chunks
+{
-copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+// UnsafeCopyMemory page error: continue after ucm
+UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
+// Copy in multi-bytes chunks
+copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+}
 __ BIND(L_exit);
 bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count);
 restore_arg_regs_using_thread();
 if (is_oop) {
 }
 BasicType type = is_oop ? T_OBJECT : T_LONG;
 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
 bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
+{
-__ jmp(L_copy_bytes);
+// UnsafeCopyMemory page error: continue after ucm
+UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
-// Copy trailing qwords
-__ BIND(L_copy_8_bytes);
+__ jmp(L_copy_bytes);
-__ movq(rax, Address(from, qword_count, Address::times_8, -8));
-__ movq(Address(to, qword_count, Address::times_8, -8), rax);
+// Copy trailing qwords
-__ decrement(qword_count);
+__ BIND(L_copy_8_bytes);
-__ jcc(Assembler::notZero, L_copy_8_bytes);
+__ movq(rax, Address(from, qword_count, Address::times_8, -8));
+__ movq(Address(to, qword_count, Address::times_8, -8), rax);
+__ decrement(qword_count);
+__ jcc(Assembler::notZero, L_copy_8_bytes);
+}
 if (is_oop) {
 __ jmp(L_exit);
 } else {
 restore_arg_regs_using_thread();
 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
 __ xorptr(rax, rax); // return 0
 __ vzeroupper();
 __ leave(); // required for proper stackwalking of RuntimeStub frame
 __ ret(0);
 }
+{
-// Copy in multi-bytes chunks
+// UnsafeCopyMemory page error: continue after ucm
-copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
+// Copy in multi-bytes chunks
+copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
+}
 __ BIND(L_exit);
 bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count);
 restore_arg_regs_using_thread();
 if (is_oop) {
 inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
 __ BIND(L_failed);
 __ xorptr(rax, rax);
 __ notptr(rax); // return -1
 __ leave();   // required for proper stackwalking of RuntimeStub frame
+__ ret(0);
+return start;
+}
+address generate_data_cache_writeback() {
+const Register src        = c_rarg0;  // source address
+__ align(CodeEntryAlignment);
+StubCodeMark mark(this, "StubRoutines", "_data_cache_writeback");
+address start = __ pc();
+__ enter();
+__ cache_wb(Address(src, 0));
+__ leave();
+__ ret(0);
+return start;
+}
+address generate_data_cache_writeback_sync() {
+const Register is_pre    = c_rarg0;  // pre or post sync
+__ align(CodeEntryAlignment);
+StubCodeMark mark(this, "StubRoutines", "_data_cache_writeback_sync");
+// pre wbsync is a no-op
+// post wbsync translates to an sfence
+Label skip;
+address start = __ pc();
+__ enter();
+__ cmpl(is_pre, 0);
+__ jcc(Assembler::notEqual, skip);
+__ cache_wbsync(false);
+__ bind(skip);
+__ leave();
 __ ret(0);
 return start;
 }
 #endif
 __ leave(); // required for proper stackwalking of RuntimeStub frame
 __ ret(0);
 return start;
 }
+address generate_electronicCodeBook_encryptAESCrypt() {
+__ align(CodeEntryAlignment);
+StubCodeMark mark(this, "StubRoutines", "electronicCodeBook_encryptAESCrypt");
+address start = __ pc();
+const Register from = c_rarg0;  // source array address
+const Register to = c_rarg1;  // destination array address
+const Register key = c_rarg2;  // key array address
+const Register len = c_rarg3;  // src len (must be multiple of blocksize 16)
+__ enter(); // required for proper stackwalking of RuntimeStub frame
+__ aesecb_encrypt(from, to, key, len);
+__ leave(); // required for proper stackwalking of RuntimeStub frame
+__ ret(0);
+return start;
+}
+address generate_electronicCodeBook_decryptAESCrypt() {
+__ align(CodeEntryAlignment);
+StubCodeMark mark(this, "StubRoutines", "electronicCodeBook_decryptAESCrypt");
+address start = __ pc();
+const Register from = c_rarg0;  // source array address
+const Register to = c_rarg1;  // destination array address
+const Register key = c_rarg2;  // key array address
+const Register len = c_rarg3;  // src len (must be multiple of blocksize 16)
+__ enter(); // required for proper stackwalking of RuntimeStub frame
+__ aesecb_decrypt(from, to, key, len);
+__ leave(); // required for proper stackwalking of RuntimeStub frame
+__ ret(0);
+return start;
+}
 address generate_upper_word_mask() {
 __ align(64);
 StubCodeMark mark(this, "StubRoutines", "upper_word_mask");
 address start = __ pc();
 StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask("vector_long_sign_mask", 0x8000000000000000);
 // support for verify_oop (must happen after universe_init)
 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
+// data cache line writeback
+StubRoutines::_data_cache_writeback = generate_data_cache_writeback();
+StubRoutines::_data_cache_writeback_sync = generate_data_cache_writeback_sync();
 // arraycopy stubs used by compilers
 generate_arraycopy_stubs();
 // don't bother generating these AES intrinsic stubs unless global flag is set
 if (UseAESIntrinsics) {
 StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
 StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
 StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
 if (VM_Version::supports_vaes() &&  VM_Version::supports_avx512vl() && VM_Version::supports_avx512dq() ) {
 StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptVectorAESCrypt();
+StubRoutines::_electronicCodeBook_encryptAESCrypt = generate_electronicCodeBook_encryptAESCrypt();
+StubRoutines::_electronicCodeBook_decryptAESCrypt = generate_electronicCodeBook_decryptAESCrypt();
 } else {
 StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
 }
 }
 if (UseAESCTRIntrinsics){
 generate_initial();
 }
 }
 }; // end class declaration
+#define UCM_TABLE_MAX_ENTRIES 16
 void StubGenerator_generate(CodeBuffer* code, bool all) {
+if (UnsafeCopyMemory::_table == NULL) {
+UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES);
+}
 StubGenerator g(code, all);
 }

branch	datagramsocketimpl-branch
changeset 58678	9cf78a70fa4f
parent 54786	ebf733a324d4
child 58679	9c3209ff7550