887 decorators |= ARRAYCOPY_ALIGNED; |
887 decorators |= ARRAYCOPY_ALIGNED; |
888 } |
888 } |
889 |
889 |
890 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
890 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
891 bs->arraycopy_prologue(_masm, decorators, t, from, to, count); |
891 bs->arraycopy_prologue(_masm, decorators, t, from, to, count); |
892 |
892 { |
893 __ subptr(to, from); // to --> to_from |
893 bool add_entry = (t != T_OBJECT && (!aligned || t == T_INT)); |
894 __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element |
894 // UnsafeCopyMemory page error: continue after ucm |
895 __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp |
895 UnsafeCopyMemoryMark ucmm(this, add_entry, true); |
896 if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) { |
896 __ subptr(to, from); // to --> to_from |
897 // align source address at 4 bytes address boundary |
897 __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element |
898 if (t == T_BYTE) { |
898 __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp |
899 // One byte misalignment happens only for byte arrays |
899 if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) { |
900 __ testl(from, 1); |
900 // align source address at 4 bytes address boundary |
901 __ jccb(Assembler::zero, L_skip_align1); |
901 if (t == T_BYTE) { |
902 __ movb(rax, Address(from, 0)); |
902 // One byte misalignment happens only for byte arrays |
903 __ movb(Address(from, to_from, Address::times_1, 0), rax); |
903 __ testl(from, 1); |
904 __ increment(from); |
904 __ jccb(Assembler::zero, L_skip_align1); |
905 __ decrement(count); |
905 __ movb(rax, Address(from, 0)); |
906 __ BIND(L_skip_align1); |
906 __ movb(Address(from, to_from, Address::times_1, 0), rax); |
|
907 __ increment(from); |
|
908 __ decrement(count); |
|
909 __ BIND(L_skip_align1); |
|
910 } |
|
911 // Two bytes misalignment happens only for byte and short (char) arrays |
|
912 __ testl(from, 2); |
|
913 __ jccb(Assembler::zero, L_skip_align2); |
|
914 __ movw(rax, Address(from, 0)); |
|
915 __ movw(Address(from, to_from, Address::times_1, 0), rax); |
|
916 __ addptr(from, 2); |
|
917 __ subl(count, 1<<(shift-1)); |
|
918 __ BIND(L_skip_align2); |
907 } |
919 } |
908 // Two bytes misalignment happens only for byte and short (char) arrays |
920 if (!VM_Version::supports_mmx()) { |
909 __ testl(from, 2); |
921 __ mov(rax, count); // save 'count' |
910 __ jccb(Assembler::zero, L_skip_align2); |
922 __ shrl(count, shift); // bytes count |
911 __ movw(rax, Address(from, 0)); |
923 __ addptr(to_from, from);// restore 'to' |
912 __ movw(Address(from, to_from, Address::times_1, 0), rax); |
924 __ rep_mov(); |
913 __ addptr(from, 2); |
925 __ subptr(to_from, from);// restore 'to_from' |
914 __ subl(count, 1<<(shift-1)); |
926 __ mov(count, rax); // restore 'count' |
915 __ BIND(L_skip_align2); |
927 __ jmpb(L_copy_2_bytes); // all dwords were copied |
916 } |
928 } else { |
917 if (!VM_Version::supports_mmx()) { |
929 if (!UseUnalignedLoadStores) { |
918 __ mov(rax, count); // save 'count' |
930 // align to 8 bytes, we know we are 4 byte aligned to start |
919 __ shrl(count, shift); // bytes count |
931 __ testptr(from, 4); |
920 __ addptr(to_from, from);// restore 'to' |
932 __ jccb(Assembler::zero, L_copy_64_bytes); |
921 __ rep_mov(); |
933 __ movl(rax, Address(from, 0)); |
922 __ subptr(to_from, from);// restore 'to_from' |
934 __ movl(Address(from, to_from, Address::times_1, 0), rax); |
923 __ mov(count, rax); // restore 'count' |
935 __ addptr(from, 4); |
924 __ jmpb(L_copy_2_bytes); // all dwords were copied |
936 __ subl(count, 1<<shift); |
925 } else { |
937 } |
926 if (!UseUnalignedLoadStores) { |
938 __ BIND(L_copy_64_bytes); |
927 // align to 8 bytes, we know we are 4 byte aligned to start |
939 __ mov(rax, count); |
928 __ testptr(from, 4); |
940 __ shrl(rax, shift+1); // 8 bytes chunk count |
929 __ jccb(Assembler::zero, L_copy_64_bytes); |
941 // |
930 __ movl(rax, Address(from, 0)); |
942 // Copy 8-byte chunks through MMX registers, 8 per iteration of the loop |
931 __ movl(Address(from, to_from, Address::times_1, 0), rax); |
943 // |
|
944 if (UseXMMForArrayCopy) { |
|
945 xmm_copy_forward(from, to_from, rax); |
|
946 } else { |
|
947 mmx_copy_forward(from, to_from, rax); |
|
948 } |
|
949 } |
|
950 // copy tailing dword |
|
951 __ BIND(L_copy_4_bytes); |
|
952 __ testl(count, 1<<shift); |
|
953 __ jccb(Assembler::zero, L_copy_2_bytes); |
|
954 __ movl(rax, Address(from, 0)); |
|
955 __ movl(Address(from, to_from, Address::times_1, 0), rax); |
|
956 if (t == T_BYTE || t == T_SHORT) { |
932 __ addptr(from, 4); |
957 __ addptr(from, 4); |
933 __ subl(count, 1<<shift); |
958 __ BIND(L_copy_2_bytes); |
|
959 // copy tailing word |
|
960 __ testl(count, 1<<(shift-1)); |
|
961 __ jccb(Assembler::zero, L_copy_byte); |
|
962 __ movw(rax, Address(from, 0)); |
|
963 __ movw(Address(from, to_from, Address::times_1, 0), rax); |
|
964 if (t == T_BYTE) { |
|
965 __ addptr(from, 2); |
|
966 __ BIND(L_copy_byte); |
|
967 // copy tailing byte |
|
968 __ testl(count, 1); |
|
969 __ jccb(Assembler::zero, L_exit); |
|
970 __ movb(rax, Address(from, 0)); |
|
971 __ movb(Address(from, to_from, Address::times_1, 0), rax); |
|
972 __ BIND(L_exit); |
|
973 } else { |
|
974 __ BIND(L_copy_byte); |
|
975 } |
|
976 } else { |
|
977 __ BIND(L_copy_2_bytes); |
934 } |
978 } |
935 __ BIND(L_copy_64_bytes); |
979 } |
936 __ mov(rax, count); |
980 |
937 __ shrl(rax, shift+1); // 8 bytes chunk count |
981 if (VM_Version::supports_mmx() && !UseXMMForArrayCopy) { |
938 // |
982 __ emms(); |
939 // Copy 8-byte chunks through MMX registers, 8 per iteration of the loop |
983 } |
940 // |
|
941 if (UseXMMForArrayCopy) { |
|
942 xmm_copy_forward(from, to_from, rax); |
|
943 } else { |
|
944 mmx_copy_forward(from, to_from, rax); |
|
945 } |
|
946 } |
|
947 // copy tailing dword |
|
948 __ BIND(L_copy_4_bytes); |
|
949 __ testl(count, 1<<shift); |
|
950 __ jccb(Assembler::zero, L_copy_2_bytes); |
|
951 __ movl(rax, Address(from, 0)); |
|
952 __ movl(Address(from, to_from, Address::times_1, 0), rax); |
|
953 if (t == T_BYTE || t == T_SHORT) { |
|
954 __ addptr(from, 4); |
|
955 __ BIND(L_copy_2_bytes); |
|
956 // copy tailing word |
|
957 __ testl(count, 1<<(shift-1)); |
|
958 __ jccb(Assembler::zero, L_copy_byte); |
|
959 __ movw(rax, Address(from, 0)); |
|
960 __ movw(Address(from, to_from, Address::times_1, 0), rax); |
|
961 if (t == T_BYTE) { |
|
962 __ addptr(from, 2); |
|
963 __ BIND(L_copy_byte); |
|
964 // copy tailing byte |
|
965 __ testl(count, 1); |
|
966 __ jccb(Assembler::zero, L_exit); |
|
967 __ movb(rax, Address(from, 0)); |
|
968 __ movb(Address(from, to_from, Address::times_1, 0), rax); |
|
969 __ BIND(L_exit); |
|
970 } else { |
|
971 __ BIND(L_copy_byte); |
|
972 } |
|
973 } else { |
|
974 __ BIND(L_copy_2_bytes); |
|
975 } |
|
976 |
|
977 __ movl(count, Address(rsp, 12+12)); // reread 'count' |
984 __ movl(count, Address(rsp, 12+12)); // reread 'count' |
978 bs->arraycopy_epilogue(_masm, decorators, t, from, to, count); |
985 bs->arraycopy_epilogue(_masm, decorators, t, from, to, count); |
979 |
986 |
980 if (t == T_OBJECT) { |
987 if (t == T_OBJECT) { |
981 __ BIND(L_0_count); |
988 __ BIND(L_0_count); |
1077 } |
1084 } |
1078 |
1085 |
1079 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
1086 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
1080 bs->arraycopy_prologue(_masm, decorators, t, from, to, count); |
1087 bs->arraycopy_prologue(_masm, decorators, t, from, to, count); |
1081 |
1088 |
1082 // copy from high to low |
1089 { |
1083 __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element |
1090 bool add_entry = (t != T_OBJECT && (!aligned || t == T_INT)); |
1084 __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp |
1091 // UnsafeCopyMemory page error: continue after ucm |
1085 if (t == T_BYTE || t == T_SHORT) { |
1092 UnsafeCopyMemoryMark ucmm(this, add_entry, true); |
1086 // Align the end of destination array at 4 bytes address boundary |
1093 // copy from high to low |
1087 __ lea(end, Address(dst, count, sf, 0)); |
1094 __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element |
1088 if (t == T_BYTE) { |
1095 __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp |
1089 // One byte misalignment happens only for byte arrays |
1096 if (t == T_BYTE || t == T_SHORT) { |
1090 __ testl(end, 1); |
1097 // Align the end of destination array at 4 bytes address boundary |
1091 __ jccb(Assembler::zero, L_skip_align1); |
1098 __ lea(end, Address(dst, count, sf, 0)); |
1092 __ decrement(count); |
1099 if (t == T_BYTE) { |
1093 __ movb(rdx, Address(from, count, sf, 0)); |
1100 // One byte misalignment happens only for byte arrays |
1094 __ movb(Address(to, count, sf, 0), rdx); |
1101 __ testl(end, 1); |
1095 __ BIND(L_skip_align1); |
1102 __ jccb(Assembler::zero, L_skip_align1); |
|
1103 __ decrement(count); |
|
1104 __ movb(rdx, Address(from, count, sf, 0)); |
|
1105 __ movb(Address(to, count, sf, 0), rdx); |
|
1106 __ BIND(L_skip_align1); |
|
1107 } |
|
1108 // Two bytes misalignment happens only for byte and short (char) arrays |
|
1109 __ testl(end, 2); |
|
1110 __ jccb(Assembler::zero, L_skip_align2); |
|
1111 __ subptr(count, 1<<(shift-1)); |
|
1112 __ movw(rdx, Address(from, count, sf, 0)); |
|
1113 __ movw(Address(to, count, sf, 0), rdx); |
|
1114 __ BIND(L_skip_align2); |
|
1115 __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element |
|
1116 __ jcc(Assembler::below, L_copy_4_bytes); |
1096 } |
1117 } |
1097 // Two bytes misalignment happens only for byte and short (char) arrays |
1118 |
1098 __ testl(end, 2); |
1119 if (!VM_Version::supports_mmx()) { |
1099 __ jccb(Assembler::zero, L_skip_align2); |
1120 __ std(); |
1100 __ subptr(count, 1<<(shift-1)); |
1121 __ mov(rax, count); // Save 'count' |
1101 __ movw(rdx, Address(from, count, sf, 0)); |
1122 __ mov(rdx, to); // Save 'to' |
1102 __ movw(Address(to, count, sf, 0), rdx); |
1123 __ lea(rsi, Address(from, count, sf, -4)); |
1103 __ BIND(L_skip_align2); |
1124 __ lea(rdi, Address(to , count, sf, -4)); |
1104 __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element |
1125 __ shrptr(count, shift); // bytes count |
1105 __ jcc(Assembler::below, L_copy_4_bytes); |
1126 __ rep_mov(); |
1106 } |
1127 __ cld(); |
1107 |
1128 __ mov(count, rax); // restore 'count' |
1108 if (!VM_Version::supports_mmx()) { |
1129 __ andl(count, (1<<shift)-1); // mask the number of rest elements |
1109 __ std(); |
1130 __ movptr(from, Address(rsp, 12+4)); // reread 'from' |
1110 __ mov(rax, count); // Save 'count' |
1131 __ mov(to, rdx); // restore 'to' |
1111 __ mov(rdx, to); // Save 'to' |
1132 __ jmpb(L_copy_2_bytes); // all dword were copied |
1112 __ lea(rsi, Address(from, count, sf, -4)); |
|
1113 __ lea(rdi, Address(to , count, sf, -4)); |
|
1114 __ shrptr(count, shift); // bytes count |
|
1115 __ rep_mov(); |
|
1116 __ cld(); |
|
1117 __ mov(count, rax); // restore 'count' |
|
1118 __ andl(count, (1<<shift)-1); // mask the number of rest elements |
|
1119 __ movptr(from, Address(rsp, 12+4)); // reread 'from' |
|
1120 __ mov(to, rdx); // restore 'to' |
|
1121 __ jmpb(L_copy_2_bytes); // all dword were copied |
|
1122 } else { |
|
1123 // Align to 8 bytes the end of array. It is aligned to 4 bytes already. |
|
1124 __ testptr(end, 4); |
|
1125 __ jccb(Assembler::zero, L_copy_8_bytes); |
|
1126 __ subl(count, 1<<shift); |
|
1127 __ movl(rdx, Address(from, count, sf, 0)); |
|
1128 __ movl(Address(to, count, sf, 0), rdx); |
|
1129 __ jmpb(L_copy_8_bytes); |
|
1130 |
|
1131 __ align(OptoLoopAlignment); |
|
1132 // Move 8 bytes |
|
1133 __ BIND(L_copy_8_bytes_loop); |
|
1134 if (UseXMMForArrayCopy) { |
|
1135 __ movq(xmm0, Address(from, count, sf, 0)); |
|
1136 __ movq(Address(to, count, sf, 0), xmm0); |
|
1137 } else { |
1133 } else { |
1138 __ movq(mmx0, Address(from, count, sf, 0)); |
1134 // Align to 8 bytes the end of array. It is aligned to 4 bytes already. |
1139 __ movq(Address(to, count, sf, 0), mmx0); |
1135 __ testptr(end, 4); |
|
1136 __ jccb(Assembler::zero, L_copy_8_bytes); |
|
1137 __ subl(count, 1<<shift); |
|
1138 __ movl(rdx, Address(from, count, sf, 0)); |
|
1139 __ movl(Address(to, count, sf, 0), rdx); |
|
1140 __ jmpb(L_copy_8_bytes); |
|
1141 |
|
1142 __ align(OptoLoopAlignment); |
|
1143 // Move 8 bytes |
|
1144 __ BIND(L_copy_8_bytes_loop); |
|
1145 if (UseXMMForArrayCopy) { |
|
1146 __ movq(xmm0, Address(from, count, sf, 0)); |
|
1147 __ movq(Address(to, count, sf, 0), xmm0); |
|
1148 } else { |
|
1149 __ movq(mmx0, Address(from, count, sf, 0)); |
|
1150 __ movq(Address(to, count, sf, 0), mmx0); |
|
1151 } |
|
1152 __ BIND(L_copy_8_bytes); |
|
1153 __ subl(count, 2<<shift); |
|
1154 __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop); |
|
1155 __ addl(count, 2<<shift); |
|
1156 if (!UseXMMForArrayCopy) { |
|
1157 __ emms(); |
|
1158 } |
1140 } |
1159 } |
1141 __ BIND(L_copy_8_bytes); |
1160 __ BIND(L_copy_4_bytes); |
1142 __ subl(count, 2<<shift); |
1161 // copy prefix qword |
1143 __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop); |
1162 __ testl(count, 1<<shift); |
1144 __ addl(count, 2<<shift); |
1163 __ jccb(Assembler::zero, L_copy_2_bytes); |
1145 if (!UseXMMForArrayCopy) { |
1164 __ movl(rdx, Address(from, count, sf, -4)); |
1146 __ emms(); |
1165 __ movl(Address(to, count, sf, -4), rdx); |
|
1166 |
|
1167 if (t == T_BYTE || t == T_SHORT) { |
|
1168 __ subl(count, (1<<shift)); |
|
1169 __ BIND(L_copy_2_bytes); |
|
1170 // copy prefix dword |
|
1171 __ testl(count, 1<<(shift-1)); |
|
1172 __ jccb(Assembler::zero, L_copy_byte); |
|
1173 __ movw(rdx, Address(from, count, sf, -2)); |
|
1174 __ movw(Address(to, count, sf, -2), rdx); |
|
1175 if (t == T_BYTE) { |
|
1176 __ subl(count, 1<<(shift-1)); |
|
1177 __ BIND(L_copy_byte); |
|
1178 // copy prefix byte |
|
1179 __ testl(count, 1); |
|
1180 __ jccb(Assembler::zero, L_exit); |
|
1181 __ movb(rdx, Address(from, 0)); |
|
1182 __ movb(Address(to, 0), rdx); |
|
1183 __ BIND(L_exit); |
|
1184 } else { |
|
1185 __ BIND(L_copy_byte); |
|
1186 } |
|
1187 } else { |
|
1188 __ BIND(L_copy_2_bytes); |
1147 } |
1189 } |
1148 } |
1190 } |
1149 __ BIND(L_copy_4_bytes); |
1191 |
1150 // copy prefix qword |
1192 if (VM_Version::supports_mmx() && !UseXMMForArrayCopy) { |
1151 __ testl(count, 1<<shift); |
1193 __ emms(); |
1152 __ jccb(Assembler::zero, L_copy_2_bytes); |
1194 } |
1153 __ movl(rdx, Address(from, count, sf, -4)); |
|
1154 __ movl(Address(to, count, sf, -4), rdx); |
|
1155 |
|
1156 if (t == T_BYTE || t == T_SHORT) { |
|
1157 __ subl(count, (1<<shift)); |
|
1158 __ BIND(L_copy_2_bytes); |
|
1159 // copy prefix dword |
|
1160 __ testl(count, 1<<(shift-1)); |
|
1161 __ jccb(Assembler::zero, L_copy_byte); |
|
1162 __ movw(rdx, Address(from, count, sf, -2)); |
|
1163 __ movw(Address(to, count, sf, -2), rdx); |
|
1164 if (t == T_BYTE) { |
|
1165 __ subl(count, 1<<(shift-1)); |
|
1166 __ BIND(L_copy_byte); |
|
1167 // copy prefix byte |
|
1168 __ testl(count, 1); |
|
1169 __ jccb(Assembler::zero, L_exit); |
|
1170 __ movb(rdx, Address(from, 0)); |
|
1171 __ movb(Address(to, 0), rdx); |
|
1172 __ BIND(L_exit); |
|
1173 } else { |
|
1174 __ BIND(L_copy_byte); |
|
1175 } |
|
1176 } else { |
|
1177 __ BIND(L_copy_2_bytes); |
|
1178 } |
|
1179 |
|
1180 __ movl2ptr(count, Address(rsp, 12+12)); // reread count |
1195 __ movl2ptr(count, Address(rsp, 12+12)); // reread count |
1181 bs->arraycopy_epilogue(_masm, decorators, t, from, to, count); |
1196 bs->arraycopy_epilogue(_masm, decorators, t, from, to, count); |
1182 |
1197 |
1183 if (t == T_OBJECT) { |
1198 if (t == T_OBJECT) { |
1184 __ BIND(L_0_count); |
1199 __ BIND(L_0_count); |
1210 __ movl2ptr(count, Address(rsp, 8+8)); // count |
1225 __ movl2ptr(count, Address(rsp, 8+8)); // count |
1211 |
1226 |
1212 *entry = __ pc(); // Entry point from conjoint arraycopy stub. |
1227 *entry = __ pc(); // Entry point from conjoint arraycopy stub. |
1213 BLOCK_COMMENT("Entry:"); |
1228 BLOCK_COMMENT("Entry:"); |
1214 |
1229 |
1215 __ subptr(to, from); // to --> to_from |
1230 { |
1216 if (VM_Version::supports_mmx()) { |
1231 // UnsafeCopyMemory page error: continue after ucm |
1217 if (UseXMMForArrayCopy) { |
1232 UnsafeCopyMemoryMark ucmm(this, true, true); |
1218 xmm_copy_forward(from, to_from, count); |
1233 __ subptr(to, from); // to --> to_from |
|
1234 if (VM_Version::supports_mmx()) { |
|
1235 if (UseXMMForArrayCopy) { |
|
1236 xmm_copy_forward(from, to_from, count); |
|
1237 } else { |
|
1238 mmx_copy_forward(from, to_from, count); |
|
1239 } |
1219 } else { |
1240 } else { |
1220 mmx_copy_forward(from, to_from, count); |
1241 __ jmpb(L_copy_8_bytes); |
|
1242 __ align(OptoLoopAlignment); |
|
1243 __ BIND(L_copy_8_bytes_loop); |
|
1244 __ fild_d(Address(from, 0)); |
|
1245 __ fistp_d(Address(from, to_from, Address::times_1)); |
|
1246 __ addptr(from, 8); |
|
1247 __ BIND(L_copy_8_bytes); |
|
1248 __ decrement(count); |
|
1249 __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop); |
1221 } |
1250 } |
1222 } else { |
1251 } |
1223 __ jmpb(L_copy_8_bytes); |
1252 if (VM_Version::supports_mmx() && !UseXMMForArrayCopy) { |
1224 __ align(OptoLoopAlignment); |
1253 __ emms(); |
1225 __ BIND(L_copy_8_bytes_loop); |
|
1226 __ fild_d(Address(from, 0)); |
|
1227 __ fistp_d(Address(from, to_from, Address::times_1)); |
|
1228 __ addptr(from, 8); |
|
1229 __ BIND(L_copy_8_bytes); |
|
1230 __ decrement(count); |
|
1231 __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop); |
|
1232 } |
1254 } |
1233 inc_copy_counter_np(T_LONG); |
1255 inc_copy_counter_np(T_LONG); |
1234 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1256 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1235 __ vzeroupper(); |
1257 __ vzeroupper(); |
1236 __ xorptr(rax, rax); // return 0 |
1258 __ xorptr(rax, rax); // return 0 |
1265 __ lea(end_from, Address(from, count, Address::times_8, 0)); |
1287 __ lea(end_from, Address(from, count, Address::times_8, 0)); |
1266 __ cmpptr(to, end_from); |
1288 __ cmpptr(to, end_from); |
1267 __ movptr(from, Address(rsp, 8)); // from |
1289 __ movptr(from, Address(rsp, 8)); // from |
1268 __ jump_cc(Assembler::aboveEqual, nooverlap); |
1290 __ jump_cc(Assembler::aboveEqual, nooverlap); |
1269 |
1291 |
1270 __ jmpb(L_copy_8_bytes); |
1292 { |
1271 |
1293 // UnsafeCopyMemory page error: continue after ucm |
1272 __ align(OptoLoopAlignment); |
1294 UnsafeCopyMemoryMark ucmm(this, true, true); |
1273 __ BIND(L_copy_8_bytes_loop); |
1295 |
1274 if (VM_Version::supports_mmx()) { |
1296 __ jmpb(L_copy_8_bytes); |
1275 if (UseXMMForArrayCopy) { |
1297 |
1276 __ movq(xmm0, Address(from, count, Address::times_8)); |
1298 __ align(OptoLoopAlignment); |
1277 __ movq(Address(to, count, Address::times_8), xmm0); |
1299 __ BIND(L_copy_8_bytes_loop); |
|
1300 if (VM_Version::supports_mmx()) { |
|
1301 if (UseXMMForArrayCopy) { |
|
1302 __ movq(xmm0, Address(from, count, Address::times_8)); |
|
1303 __ movq(Address(to, count, Address::times_8), xmm0); |
|
1304 } else { |
|
1305 __ movq(mmx0, Address(from, count, Address::times_8)); |
|
1306 __ movq(Address(to, count, Address::times_8), mmx0); |
|
1307 } |
1278 } else { |
1308 } else { |
1279 __ movq(mmx0, Address(from, count, Address::times_8)); |
1309 __ fild_d(Address(from, count, Address::times_8)); |
1280 __ movq(Address(to, count, Address::times_8), mmx0); |
1310 __ fistp_d(Address(to, count, Address::times_8)); |
1281 } |
1311 } |
1282 } else { |
1312 __ BIND(L_copy_8_bytes); |
1283 __ fild_d(Address(from, count, Address::times_8)); |
1313 __ decrement(count); |
1284 __ fistp_d(Address(to, count, Address::times_8)); |
1314 __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop); |
1285 } |
1315 |
1286 __ BIND(L_copy_8_bytes); |
1316 } |
1287 __ decrement(count); |
|
1288 __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop); |
|
1289 |
|
1290 if (VM_Version::supports_mmx() && !UseXMMForArrayCopy) { |
1317 if (VM_Version::supports_mmx() && !UseXMMForArrayCopy) { |
1291 __ emms(); |
1318 __ emms(); |
1292 } |
1319 } |
1293 inc_copy_counter_np(T_LONG); |
1320 inc_copy_counter_np(T_LONG); |
1294 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1321 __ leave(); // required for proper stackwalking of RuntimeStub frame |