hotspot/src/cpu/x86/vm/assembler_x86.cpp
changeset 34162 16b54851eaf6
parent 33628 09241459a8b8
child 34203 6817dadf6c7e
equal deleted inserted replaced
34159:f401f5b4327e 34162:16b54851eaf6
   311   // apply the compression to the displacment iff the result is8bit.
   311   // apply the compression to the displacment iff the result is8bit.
   312   if (VM_Version::supports_evex() && is_evex_inst) {
   312   if (VM_Version::supports_evex() && is_evex_inst) {
   313     switch (cur_tuple_type) {
   313     switch (cur_tuple_type) {
   314     case EVEX_FV:
   314     case EVEX_FV:
   315       if ((cur_encoding & VEX_W) == VEX_W) {
   315       if ((cur_encoding & VEX_W) == VEX_W) {
   316         mod_idx += 2 + ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
   316         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
   317       } else {
   317       } else {
   318         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
   318         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
   319       }
   319       }
   320       break;
   320       break;
   321 
   321 
   392 
   392 
   393 bool Assembler::emit_compressed_disp_byte(int &disp) {
   393 bool Assembler::emit_compressed_disp_byte(int &disp) {
   394   int mod_idx = 0;
   394   int mod_idx = 0;
   395   // We will test if the displacement fits the compressed format and if so
   395   // We will test if the displacement fits the compressed format and if so
   396   // apply the compression to the displacment iff the result is8bit.
   396   // apply the compression to the displacment iff the result is8bit.
   397   if (VM_Version::supports_evex() && _is_evex_instruction) {
   397   if (VM_Version::supports_evex() && (_attributes != NULL) && _attributes->is_evex_instruction()) {
   398     switch (_tuple_type) {
   398     int evex_encoding = _attributes->get_evex_encoding();
       
   399     int tuple_type = _attributes->get_tuple_type();
       
   400     switch (tuple_type) {
   399     case EVEX_FV:
   401     case EVEX_FV:
   400       if ((_evex_encoding & VEX_W) == VEX_W) {
   402       if ((evex_encoding & VEX_W) == VEX_W) {
   401         mod_idx += 2 + ((_evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
   403         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
   402       } else {
   404       } else {
   403         mod_idx = ((_evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
   405         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
   404       }
   406       }
   405       break;
   407       break;
   406 
   408 
   407     case EVEX_HV:
   409     case EVEX_HV:
   408       mod_idx = ((_evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
   410       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
   409       break;
   411       break;
   410 
   412 
   411     case EVEX_FVM:
   413     case EVEX_FVM:
   412       break;
   414       break;
   413 
   415 
   414     case EVEX_T1S:
   416     case EVEX_T1S:
   415       switch (_input_size_in_bits) {
   417       switch (_attributes->get_input_size()) {
   416       case EVEX_8bit:
   418       case EVEX_8bit:
   417         break;
   419         break;
   418 
   420 
   419       case EVEX_16bit:
   421       case EVEX_16bit:
   420         mod_idx = 1;
   422         mod_idx = 1;
   431       break;
   433       break;
   432 
   434 
   433     case EVEX_T1F:
   435     case EVEX_T1F:
   434     case EVEX_T2:
   436     case EVEX_T2:
   435     case EVEX_T4:
   437     case EVEX_T4:
   436       mod_idx = (_input_size_in_bits == EVEX_64bit) ? 1 : 0;
   438       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
   437       break;
   439       break;
   438 
   440 
   439     case EVEX_T8:
   441     case EVEX_T8:
   440       break;
   442       break;
   441 
   443 
   457     default:
   459     default:
   458       assert(0, "no valid evex tuple_table entry");
   460       assert(0, "no valid evex tuple_table entry");
   459       break;
   461       break;
   460     }
   462     }
   461 
   463 
   462     if (_avx_vector_len >= AVX_128bit && _avx_vector_len <= AVX_512bit) {
   464     int vector_len = _attributes->get_vector_len();
   463       int disp_factor = tuple_table[_tuple_type + mod_idx][_avx_vector_len];
   465     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
       
   466       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
   464       if ((disp % disp_factor) == 0) {
   467       if ((disp % disp_factor) == 0) {
   465         int new_disp = disp / disp_factor;
   468         int new_disp = disp / disp_factor;
   466         if (is8bit(new_disp)) {
   469         if (is8bit(new_disp)) {
   467           disp = new_disp;
   470           disp = new_disp;
   468         }
   471         }
   589       emit_int8(0x04 | regenc);
   592       emit_int8(0x04 | regenc);
   590       emit_int8(0x25);
   593       emit_int8(0x25);
   591       emit_data(disp, rspec, disp32_operand);
   594       emit_data(disp, rspec, disp32_operand);
   592     }
   595     }
   593   }
   596   }
   594   _is_evex_instruction = false;
       
   595 }
   597 }
   596 
   598 
   597 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
   599 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
   598                              Address::ScaleFactor scale, int disp,
   600                              Address::ScaleFactor scale, int disp,
   599                              RelocationHolder const& rspec) {
   601                              RelocationHolder const& rspec) {
   768     case 0x2F: // comiss
   770     case 0x2F: // comiss
   769     case 0x54: // andps
   771     case 0x54: // andps
   770     case 0x55: // andnps
   772     case 0x55: // andnps
   771     case 0x56: // orps
   773     case 0x56: // orps
   772     case 0x57: // xorps
   774     case 0x57: // xorps
   773     case 0x59: //mulpd
   775     case 0x59: // mulpd
   774     case 0x6E: // movd
   776     case 0x6E: // movd
   775     case 0x7E: // movd
   777     case 0x7E: // movd
   776     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
   778     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
   777       debug_only(has_disp32 = true);
   779       debug_only(has_disp32 = true);
   778       break;
   780       break;
  1232   emit_int32(0);   // 32-bits offset (4 bytes)
  1234   emit_int32(0);   // 32-bits offset (4 bytes)
  1233 }
  1235 }
  1234 
  1236 
  1235 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
  1237 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
  1236   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1238   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1237   if (VM_Version::supports_evex()) {
  1239   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  1238     emit_simd_arith_q(0x58, dst, src, VEX_SIMD_F2);
  1240   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  1239   } else {
  1241   emit_int8(0x58);
  1240     emit_simd_arith(0x58, dst, src, VEX_SIMD_F2);
  1242   emit_int8((unsigned char)(0xC0 | encode));
  1241   }
       
  1242 }
  1243 }
  1243 
  1244 
  1244 void Assembler::addsd(XMMRegister dst, Address src) {
  1245 void Assembler::addsd(XMMRegister dst, Address src) {
  1245   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1246   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1246   if (VM_Version::supports_evex()) {
  1247   InstructionMark im(this);
  1247     _tuple_type = EVEX_T1S;
  1248   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  1248     _input_size_in_bits = EVEX_64bit;
  1249   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  1249     emit_simd_arith_q(0x58, dst, src, VEX_SIMD_F2);
  1250   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  1250   } else {
  1251   emit_int8(0x58);
  1251     emit_simd_arith(0x58, dst, src, VEX_SIMD_F2);
  1252   emit_operand(dst, src);
  1252   }
       
  1253 }
  1253 }
  1254 
  1254 
  1255 void Assembler::addss(XMMRegister dst, XMMRegister src) {
  1255 void Assembler::addss(XMMRegister dst, XMMRegister src) {
  1256   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1256   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1257   emit_simd_arith(0x58, dst, src, VEX_SIMD_F3);
  1257   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
       
  1258   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
       
  1259   emit_int8(0x58);
       
  1260   emit_int8((unsigned char)(0xC0 | encode));
  1258 }
  1261 }
  1259 
  1262 
  1260 void Assembler::addss(XMMRegister dst, Address src) {
  1263 void Assembler::addss(XMMRegister dst, Address src) {
  1261   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1264   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1262   if (VM_Version::supports_evex()) {
  1265   InstructionMark im(this);
  1263     _tuple_type = EVEX_T1S;
  1266   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  1264     _input_size_in_bits = EVEX_32bit;
  1267   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
  1265   }
  1268   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  1266   emit_simd_arith(0x58, dst, src, VEX_SIMD_F3);
  1269   emit_int8(0x58);
       
  1270   emit_operand(dst, src);
  1267 }
  1271 }
  1268 
  1272 
  1269 void Assembler::aesdec(XMMRegister dst, Address src) {
  1273 void Assembler::aesdec(XMMRegister dst, Address src) {
  1270   assert(VM_Version::supports_aes(), "");
  1274   assert(VM_Version::supports_aes(), "");
  1271   InstructionMark im(this);
  1275   InstructionMark im(this);
  1272   simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
  1276   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  1273               VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
  1277   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  1274   emit_int8((unsigned char)0xDE);
  1278   emit_int8((unsigned char)0xDE);
  1275   emit_operand(dst, src);
  1279   emit_operand(dst, src);
  1276 }
  1280 }
  1277 
  1281 
  1278 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
  1282 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
  1279   assert(VM_Version::supports_aes(), "");
  1283   assert(VM_Version::supports_aes(), "");
  1280   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
  1284   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  1281                                       VEX_OPCODE_0F_38,  /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
  1285   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  1282   emit_int8((unsigned char)0xDE);
  1286   emit_int8((unsigned char)0xDE);
  1283   emit_int8(0xC0 | encode);
  1287   emit_int8(0xC0 | encode);
  1284 }
  1288 }
  1285 
  1289 
  1286 void Assembler::aesdeclast(XMMRegister dst, Address src) {
  1290 void Assembler::aesdeclast(XMMRegister dst, Address src) {
  1287   assert(VM_Version::supports_aes(), "");
  1291   assert(VM_Version::supports_aes(), "");
  1288   InstructionMark im(this);
  1292   InstructionMark im(this);
  1289   simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
  1293   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  1290               VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit,  /* legacy_mode */ true);
  1294   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  1291   emit_int8((unsigned char)0xDF);
  1295   emit_int8((unsigned char)0xDF);
  1292   emit_operand(dst, src);
  1296   emit_operand(dst, src);
  1293 }
  1297 }
  1294 
  1298 
  1295 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
  1299 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
  1296   assert(VM_Version::supports_aes(), "");
  1300   assert(VM_Version::supports_aes(), "");
  1297   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
  1301   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  1298                                       VEX_OPCODE_0F_38,  /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
  1302   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  1299   emit_int8((unsigned char)0xDF);
  1303   emit_int8((unsigned char)0xDF);
  1300   emit_int8((unsigned char)(0xC0 | encode));
  1304   emit_int8((unsigned char)(0xC0 | encode));
  1301 }
  1305 }
  1302 
  1306 
  1303 void Assembler::aesenc(XMMRegister dst, Address src) {
  1307 void Assembler::aesenc(XMMRegister dst, Address src) {
  1304   assert(VM_Version::supports_aes(), "");
  1308   assert(VM_Version::supports_aes(), "");
  1305   InstructionMark im(this);
  1309   InstructionMark im(this);
  1306   simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
  1310   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  1307               VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
  1311   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  1308   emit_int8((unsigned char)0xDC);
  1312   emit_int8((unsigned char)0xDC);
  1309   emit_operand(dst, src);
  1313   emit_operand(dst, src);
  1310 }
  1314 }
  1311 
  1315 
  1312 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
  1316 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
  1313   assert(VM_Version::supports_aes(), "");
  1317   assert(VM_Version::supports_aes(), "");
  1314   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
  1318   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  1315                                       VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
  1319   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  1316   emit_int8((unsigned char)0xDC);
  1320   emit_int8((unsigned char)0xDC);
  1317   emit_int8(0xC0 | encode);
  1321   emit_int8(0xC0 | encode);
  1318 }
  1322 }
  1319 
  1323 
  1320 void Assembler::aesenclast(XMMRegister dst, Address src) {
  1324 void Assembler::aesenclast(XMMRegister dst, Address src) {
  1321   assert(VM_Version::supports_aes(), "");
  1325   assert(VM_Version::supports_aes(), "");
  1322   InstructionMark im(this);
  1326   InstructionMark im(this);
  1323   simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
  1327   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  1324               VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit,  /* legacy_mode */ true);
  1328   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  1325   emit_int8((unsigned char)0xDD);
  1329   emit_int8((unsigned char)0xDD);
  1326   emit_operand(dst, src);
  1330   emit_operand(dst, src);
  1327 }
  1331 }
  1328 
  1332 
  1329 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
  1333 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
  1330   assert(VM_Version::supports_aes(), "");
  1334   assert(VM_Version::supports_aes(), "");
  1331   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
  1335   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  1332                                       VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
  1336   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  1333   emit_int8((unsigned char)0xDD);
  1337   emit_int8((unsigned char)0xDD);
  1334   emit_int8((unsigned char)(0xC0 | encode));
  1338   emit_int8((unsigned char)(0xC0 | encode));
  1335 }
  1339 }
  1336 
  1340 
  1337 void Assembler::andl(Address dst, int32_t imm32) {
  1341 void Assembler::andl(Address dst, int32_t imm32) {
  1359   emit_arith(0x23, 0xC0, dst, src);
  1363   emit_arith(0x23, 0xC0, dst, src);
  1360 }
  1364 }
  1361 
  1365 
  1362 void Assembler::andnl(Register dst, Register src1, Register src2) {
  1366 void Assembler::andnl(Register dst, Register src1, Register src2) {
  1363   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  1367   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  1364   int encode = vex_prefix_0F38_and_encode_legacy(dst, src1, src2);
  1368   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  1369   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
  1365   emit_int8((unsigned char)0xF2);
  1370   emit_int8((unsigned char)0xF2);
  1366   emit_int8((unsigned char)(0xC0 | encode));
  1371   emit_int8((unsigned char)(0xC0 | encode));
  1367 }
  1372 }
  1368 
  1373 
  1369 void Assembler::andnl(Register dst, Register src1, Address src2) {
  1374 void Assembler::andnl(Register dst, Register src1, Address src2) {
  1370   InstructionMark im(this);
       
  1371   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  1375   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  1372   vex_prefix_0F38_legacy(dst, src1, src2);
  1376   InstructionMark im(this);
       
  1377   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  1378   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
  1373   emit_int8((unsigned char)0xF2);
  1379   emit_int8((unsigned char)0xF2);
  1374   emit_operand(dst, src2);
  1380   emit_operand(dst, src2);
  1375 }
  1381 }
  1376 
  1382 
  1377 void Assembler::bsfl(Register dst, Register src) {
  1383 void Assembler::bsfl(Register dst, Register src) {
  1394   emit_int8((unsigned char)(0xC8 | encode));
  1400   emit_int8((unsigned char)(0xC8 | encode));
  1395 }
  1401 }
  1396 
  1402 
  1397 void Assembler::blsil(Register dst, Register src) {
  1403 void Assembler::blsil(Register dst, Register src) {
  1398   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  1404   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  1399   int encode = vex_prefix_0F38_and_encode_legacy(rbx, dst, src);
  1405   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  1406   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
  1400   emit_int8((unsigned char)0xF3);
  1407   emit_int8((unsigned char)0xF3);
  1401   emit_int8((unsigned char)(0xC0 | encode));
  1408   emit_int8((unsigned char)(0xC0 | encode));
  1402 }
  1409 }
  1403 
  1410 
  1404 void Assembler::blsil(Register dst, Address src) {
  1411 void Assembler::blsil(Register dst, Address src) {
  1405   InstructionMark im(this);
       
  1406   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  1412   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  1407   vex_prefix_0F38_legacy(rbx, dst, src);
  1413   InstructionMark im(this);
       
  1414   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  1415   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
  1408   emit_int8((unsigned char)0xF3);
  1416   emit_int8((unsigned char)0xF3);
  1409   emit_operand(rbx, src);
  1417   emit_operand(rbx, src);
  1410 }
  1418 }
  1411 
  1419 
  1412 void Assembler::blsmskl(Register dst, Register src) {
  1420 void Assembler::blsmskl(Register dst, Register src) {
  1413   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  1421   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  1414   int encode = vex_prefix_0F38_and_encode_legacy(rdx, dst, src);
  1422   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  1423   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
  1415   emit_int8((unsigned char)0xF3);
  1424   emit_int8((unsigned char)0xF3);
  1416   emit_int8((unsigned char)(0xC0 | encode));
  1425   emit_int8((unsigned char)(0xC0 | encode));
  1417 }
  1426 }
  1418 
  1427 
  1419 void Assembler::blsmskl(Register dst, Address src) {
  1428 void Assembler::blsmskl(Register dst, Address src) {
  1420   InstructionMark im(this);
       
  1421   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  1429   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  1422   vex_prefix_0F38_legacy(rdx, dst, src);
  1430   InstructionMark im(this);
       
  1431   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  1432   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
  1423   emit_int8((unsigned char)0xF3);
  1433   emit_int8((unsigned char)0xF3);
  1424   emit_operand(rdx, src);
  1434   emit_operand(rdx, src);
  1425 }
  1435 }
  1426 
  1436 
  1427 void Assembler::blsrl(Register dst, Register src) {
  1437 void Assembler::blsrl(Register dst, Register src) {
  1428   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  1438   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  1429   int encode = vex_prefix_0F38_and_encode_legacy(rcx, dst, src);
  1439   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  1440   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
  1430   emit_int8((unsigned char)0xF3);
  1441   emit_int8((unsigned char)0xF3);
  1431   emit_int8((unsigned char)(0xC0 | encode));
  1442   emit_int8((unsigned char)(0xC0 | encode));
  1432 }
  1443 }
  1433 
  1444 
  1434 void Assembler::blsrl(Register dst, Address src) {
  1445 void Assembler::blsrl(Register dst, Address src) {
  1435   InstructionMark im(this);
       
  1436   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  1446   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  1437   vex_prefix_0F38_legacy(rcx, dst, src);
  1447   InstructionMark im(this);
       
  1448   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  1449   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
  1438   emit_int8((unsigned char)0xF3);
  1450   emit_int8((unsigned char)0xF3);
  1439   emit_operand(rcx, src);
  1451   emit_operand(rcx, src);
  1440 }
  1452 }
  1441 
  1453 
  1442 void Assembler::call(Label& L, relocInfo::relocType rtype) {
  1454 void Assembler::call(Label& L, relocInfo::relocType rtype) {
  1579 
  1591 
  1580 void Assembler::comisd(XMMRegister dst, Address src) {
  1592 void Assembler::comisd(XMMRegister dst, Address src) {
  1581   // NOTE: dbx seems to decode this as comiss even though the
  1593   // NOTE: dbx seems to decode this as comiss even though the
  1582   // 0x66 is there. Strangly ucomisd comes out correct
  1594   // 0x66 is there. Strangly ucomisd comes out correct
  1583   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1595   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1584   if (VM_Version::supports_evex()) {
  1596   InstructionMark im(this);
  1585     _tuple_type = EVEX_T1S;
  1597   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
  1586     _input_size_in_bits = EVEX_64bit;
  1598   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  1587     emit_simd_arith_nonds_q(0x2F, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
  1599   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  1588   } else {
  1600   emit_int8(0x2F);
  1589     emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_66);
  1601   emit_operand(dst, src);
  1590   }
       
  1591 }
  1602 }
  1592 
  1603 
  1593 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
  1604 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
  1594   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1605   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1595   if (VM_Version::supports_evex()) {
  1606   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  1596     emit_simd_arith_nonds_q(0x2F, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
  1607   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  1597   } else {
  1608   emit_int8(0x2F);
  1598     emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_66);
  1609   emit_int8((unsigned char)(0xC0 | encode));
  1599   }
       
  1600 }
  1610 }
  1601 
  1611 
  1602 void Assembler::comiss(XMMRegister dst, Address src) {
  1612 void Assembler::comiss(XMMRegister dst, Address src) {
  1603   if (VM_Version::supports_evex()) {
       
  1604     _tuple_type = EVEX_T1S;
       
  1605     _input_size_in_bits = EVEX_32bit;
       
  1606   }
       
  1607   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1613   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1608   emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ true);
  1614   InstructionMark im(this);
       
  1615   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  1616   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
       
  1617   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  1618   emit_int8(0x2F);
       
  1619   emit_operand(dst, src);
  1609 }
  1620 }
  1610 
  1621 
  1611 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
  1622 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
  1612   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1623   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1613   emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ true);
  1624   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  1625   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  1626   emit_int8(0x2F);
       
  1627   emit_int8((unsigned char)(0xC0 | encode));
  1614 }
  1628 }
  1615 
  1629 
  1616 void Assembler::cpuid() {
  1630 void Assembler::cpuid() {
  1617   emit_int8(0x0F);
  1631   emit_int8(0x0F);
  1618   emit_int8((unsigned char)0xA2);
  1632   emit_int8((unsigned char)0xA2);
  1697   emit_operand(crc, adr);
  1711   emit_operand(crc, adr);
  1698 }
  1712 }
  1699 
  1713 
  1700 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
  1714 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
  1701   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1715   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1702   emit_simd_arith_nonds(0xE6, dst, src, VEX_SIMD_F3, /* no_mask_reg */ false, /* legacy_mode */ true);
  1716   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  1717   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
       
  1718   emit_int8((unsigned char)0xE6);
       
  1719   emit_int8((unsigned char)(0xC0 | encode));
  1703 }
  1720 }
  1704 
  1721 
  1705 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
  1722 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
  1706   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1723   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1707   emit_simd_arith_nonds(0x5B, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ true);
  1724   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  1725   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  1726   emit_int8(0x5B);
       
  1727   emit_int8((unsigned char)(0xC0 | encode));
  1708 }
  1728 }
  1709 
  1729 
  1710 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
  1730 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
  1711   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1731   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1712   if (VM_Version::supports_evex()) {
  1732   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  1713     emit_simd_arith_q(0x5A, dst, src, VEX_SIMD_F2);
  1733   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  1714   } else {
  1734   emit_int8(0x5A);
  1715     emit_simd_arith(0x5A, dst, src, VEX_SIMD_F2);
  1735   emit_int8((unsigned char)(0xC0 | encode));
  1716   }
       
  1717 }
  1736 }
  1718 
  1737 
  1719 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
  1738 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
  1720   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1739   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1721   if (VM_Version::supports_evex()) {
  1740   InstructionMark im(this);
  1722     _tuple_type = EVEX_T1F;
  1741   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  1723     _input_size_in_bits = EVEX_64bit;
  1742   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  1724     emit_simd_arith_q(0x5A, dst, src, VEX_SIMD_F2);
  1743   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  1725   } else {
  1744   emit_int8(0x5A);
  1726     emit_simd_arith(0x5A, dst, src, VEX_SIMD_F2);
  1745   emit_operand(dst, src);
  1727   }
       
  1728 }
  1746 }
  1729 
  1747 
  1730 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
  1748 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
  1731   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1749   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1732   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VM_Version::supports_evex());
  1750   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  1751   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  1733   emit_int8(0x2A);
  1752   emit_int8(0x2A);
  1734   emit_int8((unsigned char)(0xC0 | encode));
  1753   emit_int8((unsigned char)(0xC0 | encode));
  1735 }
  1754 }
  1736 
  1755 
  1737 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
  1756 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
  1738   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1757   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1739   if (VM_Version::supports_evex()) {
  1758   InstructionMark im(this);
  1740     _tuple_type = EVEX_T1S;
  1759   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  1741     _input_size_in_bits = EVEX_32bit;
  1760   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
  1742     emit_simd_arith(0x2A, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);
  1761   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  1743   } else {
  1762   emit_int8(0x2A);
  1744     emit_simd_arith(0x2A, dst, src, VEX_SIMD_F2);
  1763   emit_operand(dst, src);
  1745   }
       
  1746 }
  1764 }
  1747 
  1765 
  1748 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
  1766 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
  1749   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1767   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1750   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
  1768   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  1769   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  1751   emit_int8(0x2A);
  1770   emit_int8(0x2A);
  1752   emit_int8((unsigned char)(0xC0 | encode));
  1771   emit_int8((unsigned char)(0xC0 | encode));
  1753 }
  1772 }
  1754 
  1773 
  1755 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
  1774 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
  1756   if (VM_Version::supports_evex()) {
       
  1757     _tuple_type = EVEX_T1S;
       
  1758     _input_size_in_bits = EVEX_32bit;
       
  1759   }
       
  1760   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1775   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1761   emit_simd_arith(0x2A, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
  1776   InstructionMark im(this);
       
  1777   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  1778   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
       
  1779   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
       
  1780   emit_int8(0x2A);
       
  1781   emit_operand(dst, src);
  1762 }
  1782 }
  1763 
  1783 
  1764 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
  1784 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
  1765   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1785   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1766   int encode = simd_prefix_and_encode_q(dst, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
  1786   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  1787   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  1767   emit_int8(0x2A);
  1788   emit_int8(0x2A);
  1768   emit_int8((unsigned char)(0xC0 | encode));
  1789   emit_int8((unsigned char)(0xC0 | encode));
  1769 }
  1790 }
  1770 
  1791 
  1771 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
  1792 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
  1772   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1793   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1773   emit_simd_arith(0x5A, dst, src, VEX_SIMD_F3);
  1794   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
       
  1795   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
       
  1796   emit_int8(0x5A);
       
  1797   emit_int8((unsigned char)(0xC0 | encode));
  1774 }
  1798 }
  1775 
  1799 
  1776 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
  1800 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
  1777   if (VM_Version::supports_evex()) {
  1801   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1778     _tuple_type = EVEX_T1S;
  1802   InstructionMark im(this);
  1779     _input_size_in_bits = EVEX_32bit;
  1803   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  1780   }
  1804   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
  1781   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1805   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  1782   emit_simd_arith(0x5A, dst, src, VEX_SIMD_F3);
  1806   emit_int8(0x5A);
       
  1807   emit_operand(dst, src);
  1783 }
  1808 }
  1784 
  1809 
  1785 
  1810 
  1786 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
  1811 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
  1787   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1812   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1788   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, /* no_mask_reg */ true);
  1813   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  1814   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  1789   emit_int8(0x2C);
  1815   emit_int8(0x2C);
  1790   emit_int8((unsigned char)(0xC0 | encode));
  1816   emit_int8((unsigned char)(0xC0 | encode));
  1791 }
  1817 }
  1792 
  1818 
  1793 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
  1819 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
  1794   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1820   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1795   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, /* no_mask_reg */ true);
  1821   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  1822   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  1796   emit_int8(0x2C);
  1823   emit_int8(0x2C);
  1797   emit_int8((unsigned char)(0xC0 | encode));
  1824   emit_int8((unsigned char)(0xC0 | encode));
  1798 }
  1825 }
  1799 
  1826 
  1800 void Assembler::decl(Address dst) {
  1827 void Assembler::decl(Address dst) {
  1805   emit_operand(rcx, dst);
  1832   emit_operand(rcx, dst);
  1806 }
  1833 }
  1807 
  1834 
  1808 void Assembler::divsd(XMMRegister dst, Address src) {
  1835 void Assembler::divsd(XMMRegister dst, Address src) {
  1809   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1836   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1810   if (VM_Version::supports_evex()) {
  1837   InstructionMark im(this);
  1811     _tuple_type = EVEX_T1S;
  1838   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  1812     _input_size_in_bits = EVEX_64bit;
  1839   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  1813     emit_simd_arith_q(0x5E, dst, src, VEX_SIMD_F2);
  1840   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  1814   } else {
  1841   emit_int8(0x5E);
  1815     emit_simd_arith(0x5E, dst, src, VEX_SIMD_F2);
  1842   emit_operand(dst, src);
  1816   }
       
  1817 }
  1843 }
  1818 
  1844 
  1819 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
  1845 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
  1820   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1846   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  1821   if (VM_Version::supports_evex()) {
  1847   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  1822     emit_simd_arith_q(0x5E, dst, src, VEX_SIMD_F2);
  1848   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  1823   } else {
  1849   emit_int8(0x5E);
  1824     emit_simd_arith(0x5E, dst, src, VEX_SIMD_F2);
  1850   emit_int8((unsigned char)(0xC0 | encode));
  1825   }
       
  1826 }
  1851 }
  1827 
  1852 
  1828 void Assembler::divss(XMMRegister dst, Address src) {
  1853 void Assembler::divss(XMMRegister dst, Address src) {
  1829   if (VM_Version::supports_evex()) {
       
  1830     _tuple_type = EVEX_T1S;
       
  1831     _input_size_in_bits = EVEX_32bit;
       
  1832   }
       
  1833   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1854   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1834   emit_simd_arith(0x5E, dst, src, VEX_SIMD_F3);
  1855   InstructionMark im(this);
       
  1856   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
       
  1857   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
       
  1858   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
       
  1859   emit_int8(0x5E);
       
  1860   emit_operand(dst, src);
  1835 }
  1861 }
  1836 
  1862 
  1837 void Assembler::divss(XMMRegister dst, XMMRegister src) {
  1863 void Assembler::divss(XMMRegister dst, XMMRegister src) {
  1838   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1864   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  1839   emit_simd_arith(0x5E, dst, src, VEX_SIMD_F3);
  1865   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
       
  1866   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
       
  1867   emit_int8(0x5E);
       
  1868   emit_int8((unsigned char)(0xC0 | encode));
  1840 }
  1869 }
  1841 
  1870 
  1842 void Assembler::emms() {
  1871 void Assembler::emms() {
  1843   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
  1872   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
  1844   emit_int8(0x0F);
  1873   emit_int8(0x0F);
  2080   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
  2109   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
  2081 }
  2110 }
  2082 
  2111 
  2083 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
  2112 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
  2084   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2113   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2085   if (VM_Version::supports_avx512novl()) {
  2114   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
  2086     int vector_len = AVX_512bit;
  2115   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  2087     int dst_enc = dst->encoding();
  2116   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  2088     int src_enc = src->encoding();
  2117   emit_int8(0x28);
  2089     int encode = vex_prefix_and_encode(dst_enc, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F,
  2118   emit_int8((unsigned char)(0xC0 | encode));
  2090                                        /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
       
  2091     emit_int8(0x28);
       
  2092     emit_int8((unsigned char)(0xC0 | encode));
       
  2093   } else if (VM_Version::supports_evex()) {
       
  2094     emit_simd_arith_nonds_q(0x28, dst, src, VEX_SIMD_66);
       
  2095   } else {
       
  2096     emit_simd_arith_nonds(0x28, dst, src, VEX_SIMD_66);
       
  2097   }
       
  2098 }
  2119 }
  2099 
  2120 
  2100 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
  2121 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
  2101   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  2122   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  2102   if (VM_Version::supports_avx512novl()) {
  2123   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
  2103     int vector_len = AVX_512bit;
  2124   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  2104     int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, vector_len);
  2125   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
  2105     emit_int8(0x28);
  2126   emit_int8(0x28);
  2106     emit_int8((unsigned char)(0xC0 | encode));
  2127   emit_int8((unsigned char)(0xC0 | encode));
  2107   } else {
       
  2108     emit_simd_arith_nonds(0x28, dst, src, VEX_SIMD_NONE);
       
  2109   }
       
  2110 }
  2128 }
  2111 
  2129 
  2112 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
  2130 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
  2113   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  2131   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  2114   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, /* no_mask_reg */ true);
  2132   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  2133   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
  2115   emit_int8(0x16);
  2134   emit_int8(0x16);
  2116   emit_int8((unsigned char)(0xC0 | encode));
  2135   emit_int8((unsigned char)(0xC0 | encode));
  2117 }
  2136 }
  2118 
  2137 
  2119 void Assembler::movb(Register dst, Address src) {
  2138 void Assembler::movb(Register dst, Address src) {
  2123   emit_int8((unsigned char)0x8A);
  2142   emit_int8((unsigned char)0x8A);
  2124   emit_operand(dst, src);
  2143   emit_operand(dst, src);
  2125 }
  2144 }
  2126 
  2145 
  2127 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
  2146 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
  2128   _instruction_uses_vl = true;
       
  2129   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
  2147   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
  2130   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, /* no_mask_reg */ false, VEX_OPCODE_0F,
  2148   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_128bit;
  2131                                       /* rex_w */ VM_Version::supports_evex(), AVX_128bit, /* legacy_mode */ false);
  2149   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  2150   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  2132   emit_int8(0x12);
  2151   emit_int8(0x12);
  2133   emit_int8(0xC0 | encode);
  2152   emit_int8(0xC0 | encode);
  2134 
       
  2135 }
  2153 }
  2136 
  2154 
  2137 void Assembler::kmovql(KRegister dst, KRegister src) {
  2155 void Assembler::kmovql(KRegister dst, KRegister src) {
  2138   NOT_LP64(assert(VM_Version::supports_evex(), ""));
  2156   NOT_LP64(assert(VM_Version::supports_evex(), ""));
  2139   int encode = kreg_prefix_and_encode(dst, knoreg, src, VEX_SIMD_NONE,
  2157   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
  2140                                       /* no_mask_reg */ true, VEX_OPCODE_0F, /* rex_w */ true);
  2158   int encode = kreg_prefix_and_encode(dst, knoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
  2141   emit_int8((unsigned char)0x90);
  2159   emit_int8((unsigned char)0x90);
  2142   emit_int8((unsigned char)(0xC0 | encode));
  2160   emit_int8((unsigned char)(0xC0 | encode));
  2143 }
  2161 }
  2144 
  2162 
  2145 void Assembler::kmovql(KRegister dst, Address src) {
  2163 void Assembler::kmovql(KRegister dst, Address src) {
  2146   NOT_LP64(assert(VM_Version::supports_evex(), ""));
  2164   NOT_LP64(assert(VM_Version::supports_evex(), ""));
  2147   int dst_enc = dst->encoding();
  2165   InstructionMark im(this);
  2148   int nds_enc = 0;
  2166   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
  2149   vex_prefix(src, nds_enc, dst_enc, VEX_SIMD_NONE,
  2167   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
  2150              VEX_OPCODE_0F, /* vex_w */  true, AVX_128bit, /* legacy_mode */ true, /* no_reg_mask */ true);
       
  2151   emit_int8((unsigned char)0x90);
  2168   emit_int8((unsigned char)0x90);
  2152   emit_operand((Register)dst, src);
  2169   emit_operand((Register)dst, src);
  2153 }
  2170 }
  2154 
  2171 
  2155 void Assembler::kmovql(Address dst, KRegister src) {
  2172 void Assembler::kmovql(Address dst, KRegister src) {
  2156   NOT_LP64(assert(VM_Version::supports_evex(), ""));
  2173   NOT_LP64(assert(VM_Version::supports_evex(), ""));
  2157   int src_enc = src->encoding();
  2174   InstructionMark im(this);
  2158   int nds_enc = 0;
  2175   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
  2159   vex_prefix(dst, nds_enc, src_enc, VEX_SIMD_NONE,
  2176   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
  2160              VEX_OPCODE_0F, /* vex_w */ true, AVX_128bit, /* legacy_mode */ true, /* no_reg_mask */ true);
       
  2161   emit_int8((unsigned char)0x90);
  2177   emit_int8((unsigned char)0x90);
  2162   emit_operand((Register)src, dst);
  2178   emit_operand((Register)src, dst);
  2163 }
  2179 }
  2164 
  2180 
  2165 void Assembler::kmovql(KRegister dst, Register src) {
  2181 void Assembler::kmovql(KRegister dst, Register src) {
  2166   NOT_LP64(assert(VM_Version::supports_evex(), ""));
  2182   NOT_LP64(assert(VM_Version::supports_evex(), ""));
  2167   VexSimdPrefix pre = !_legacy_mode_bw ? VEX_SIMD_F2 : VEX_SIMD_NONE;
  2183   VexSimdPrefix pre = !_legacy_mode_bw ? VEX_SIMD_F2 : VEX_SIMD_NONE;
  2168   int encode = kreg_prefix_and_encode(dst, knoreg, src, pre, /* no_mask_reg */ true,
  2184   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_bw, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
  2169                                       VEX_OPCODE_0F, /* legacy_mode */ !_legacy_mode_bw);
  2185   int encode = kreg_prefix_and_encode(dst, knoreg, src, pre, VEX_OPCODE_0F, &attributes);
  2170   emit_int8((unsigned char)0x92);
  2186   emit_int8((unsigned char)0x92);
  2171   emit_int8((unsigned char)(0xC0 | encode));
  2187   emit_int8((unsigned char)(0xC0 | encode));
  2172 }
  2188 }
  2173 
  2189 
  2174 void Assembler::kmovdl(KRegister dst, Register src) {
  2190 void Assembler::kmovdl(KRegister dst, Register src) {
  2175   NOT_LP64(assert(VM_Version::supports_evex(), ""));
  2191   NOT_LP64(assert(VM_Version::supports_evex(), ""));
  2176   VexSimdPrefix pre = !_legacy_mode_bw ? VEX_SIMD_F2 : VEX_SIMD_NONE;
  2192   VexSimdPrefix pre = !_legacy_mode_bw ? VEX_SIMD_F2 : VEX_SIMD_NONE;
  2177   int encode = kreg_prefix_and_encode(dst, knoreg, src, pre, /* no_mask_reg */ true);
  2193   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
       
  2194   int encode = kreg_prefix_and_encode(dst, knoreg, src, pre, VEX_OPCODE_0F, &attributes);
  2178   emit_int8((unsigned char)0x92);
  2195   emit_int8((unsigned char)0x92);
  2179   emit_int8((unsigned char)(0xC0 | encode));
  2196   emit_int8((unsigned char)(0xC0 | encode));
  2180 }
  2197 }
  2181 
  2198 
  2182 void Assembler::kmovwl(KRegister dst, Register src) {
  2199 void Assembler::kmovwl(KRegister dst, Register src) {
  2183   NOT_LP64(assert(VM_Version::supports_evex(), ""));
  2200   NOT_LP64(assert(VM_Version::supports_evex(), ""));
  2184   int encode = kreg_prefix_and_encode(dst, knoreg, src, VEX_SIMD_NONE, /* no_mask_reg */ true);
  2201   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
       
  2202   int encode = kreg_prefix_and_encode(dst, knoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
  2185   emit_int8((unsigned char)0x92);
  2203   emit_int8((unsigned char)0x92);
  2186   emit_int8((unsigned char)(0xC0 | encode));
  2204   emit_int8((unsigned char)(0xC0 | encode));
  2187 }
  2205 }
  2188 
  2206 
  2189 void Assembler::movb(Address dst, int imm8) {
  2207 void Assembler::movb(Address dst, int imm8) {
  2203   emit_operand(src, dst);
  2221   emit_operand(src, dst);
  2204 }
  2222 }
  2205 
  2223 
  2206 void Assembler::movdl(XMMRegister dst, Register src) {
  2224 void Assembler::movdl(XMMRegister dst, Register src) {
  2207   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2225   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2208   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
  2226   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  2227   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  2209   emit_int8(0x6E);
  2228   emit_int8(0x6E);
  2210   emit_int8((unsigned char)(0xC0 | encode));
  2229   emit_int8((unsigned char)(0xC0 | encode));
  2211 }
  2230 }
  2212 
  2231 
  2213 void Assembler::movdl(Register dst, XMMRegister src) {
  2232 void Assembler::movdl(Register dst, XMMRegister src) {
  2214   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2233   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
       
  2234   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  2215   // swap src/dst to get correct prefix
  2235   // swap src/dst to get correct prefix
  2216   int encode = simd_prefix_and_encode(src, dst, VEX_SIMD_66, /* no_mask_reg */ true);
  2236   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  2217   emit_int8(0x7E);
  2237   emit_int8(0x7E);
  2218   emit_int8((unsigned char)(0xC0 | encode));
  2238   emit_int8((unsigned char)(0xC0 | encode));
  2219 }
  2239 }
  2220 
  2240 
  2221 void Assembler::movdl(XMMRegister dst, Address src) {
  2241 void Assembler::movdl(XMMRegister dst, Address src) {
  2222   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2242   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2223   if (VM_Version::supports_evex()) {
  2243   InstructionMark im(this);
  2224     _tuple_type = EVEX_T1S;
  2244   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  2225     _input_size_in_bits = EVEX_32bit;
  2245   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
  2226   }
  2246   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  2227   InstructionMark im(this);
       
  2228   simd_prefix(dst, src, VEX_SIMD_66, /* no_reg_mask */ true);
       
  2229   emit_int8(0x6E);
  2247   emit_int8(0x6E);
  2230   emit_operand(dst, src);
  2248   emit_operand(dst, src);
  2231 }
  2249 }
  2232 
  2250 
  2233 void Assembler::movdl(Address dst, XMMRegister src) {
  2251 void Assembler::movdl(Address dst, XMMRegister src) {
  2234   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2252   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2235   if (VM_Version::supports_evex()) {
  2253   InstructionMark im(this);
  2236     _tuple_type = EVEX_T1S;
  2254   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  2237     _input_size_in_bits = EVEX_32bit;
  2255   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
  2238   }
  2256   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  2239   InstructionMark im(this);
       
  2240   simd_prefix(dst, src, VEX_SIMD_66, /* no_reg_mask */ true);
       
  2241   emit_int8(0x7E);
  2257   emit_int8(0x7E);
  2242   emit_operand(src, dst);
  2258   emit_operand(src, dst);
  2243 }
  2259 }
  2244 
  2260 
  2245 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
  2261 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
  2246   _instruction_uses_vl = true;
  2262   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2247   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2263   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
  2248   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_66);
  2264   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  2265   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  2266   emit_int8(0x6F);
       
  2267   emit_int8((unsigned char)(0xC0 | encode));
  2249 }
  2268 }
  2250 
  2269 
  2251 void Assembler::movdqa(XMMRegister dst, Address src) {
  2270 void Assembler::movdqa(XMMRegister dst, Address src) {
  2252   _instruction_uses_vl = true;
  2271   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2253   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2272   InstructionMark im(this);
  2254   if (VM_Version::supports_evex()) {
  2273   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  2255     _tuple_type = EVEX_FVM;
  2274   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
  2256   }
  2275   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  2257   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_66);
  2276   emit_int8(0x6F);
       
  2277   emit_operand(dst, src);
  2258 }
  2278 }
  2259 
  2279 
  2260 void Assembler::movdqu(XMMRegister dst, Address src) {
  2280 void Assembler::movdqu(XMMRegister dst, Address src) {
  2261   _instruction_uses_vl = true;
  2281   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2262   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2282   InstructionMark im(this);
  2263   if (VM_Version::supports_evex()) {
  2283   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
  2264     _tuple_type = EVEX_FVM;
  2284   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
  2265   }
  2285   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  2266   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_F3);
  2286   emit_int8(0x6F);
       
  2287   emit_operand(dst, src);
  2267 }
  2288 }
  2268 
  2289 
  2269 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
  2290 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
  2270   _instruction_uses_vl = true;
  2291   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2271   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2292   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  2272   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_F3);
  2293   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
       
  2294   emit_int8(0x6F);
       
  2295   emit_int8((unsigned char)(0xC0 | encode));
  2273 }
  2296 }
  2274 
  2297 
  2275 void Assembler::movdqu(Address dst, XMMRegister src) {
  2298 void Assembler::movdqu(Address dst, XMMRegister src) {
  2276   _instruction_uses_vl = true;
  2299   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2277   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2300   InstructionMark im(this);
  2278   if (VM_Version::supports_evex()) {
  2301   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  2279     _tuple_type = EVEX_FVM;
  2302   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
  2280   }
  2303   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  2281   InstructionMark im(this);
       
  2282   simd_prefix(dst, src, VEX_SIMD_F3, /* no_mask_reg */ false);
       
  2283   emit_int8(0x7F);
  2304   emit_int8(0x7F);
  2284   emit_operand(src, dst);
  2305   emit_operand(src, dst);
  2285 }
  2306 }
  2286 
  2307 
  2287 // Move Unaligned 256bit Vector
  2308 // Move Unaligned 256bit Vector
  2288 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
  2309 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
  2289   _instruction_uses_vl = true;
       
  2290   assert(UseAVX > 0, "");
  2310   assert(UseAVX > 0, "");
  2291   int vector_len = AVX_256bit;
  2311   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  2292   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, vector_len);
  2312   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  2293   emit_int8(0x6F);
  2313   emit_int8(0x6F);
  2294   emit_int8((unsigned char)(0xC0 | encode));
  2314   emit_int8((unsigned char)(0xC0 | encode));
  2295 }
  2315 }
  2296 
  2316 
  2297 void Assembler::vmovdqu(XMMRegister dst, Address src) {
  2317 void Assembler::vmovdqu(XMMRegister dst, Address src) {
  2298   _instruction_uses_vl = true;
       
  2299   assert(UseAVX > 0, "");
  2318   assert(UseAVX > 0, "");
  2300   if (VM_Version::supports_evex()) {
  2319   InstructionMark im(this);
  2301     _tuple_type = EVEX_FVM;
  2320   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
  2302   }
  2321   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
  2303   InstructionMark im(this);
  2322   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  2304   int vector_len = AVX_256bit;
       
  2305   vex_prefix(dst, xnoreg, src, VEX_SIMD_F3, vector_len);
       
  2306   emit_int8(0x6F);
  2323   emit_int8(0x6F);
  2307   emit_operand(dst, src);
  2324   emit_operand(dst, src);
  2308 }
  2325 }
  2309 
  2326 
  2310 void Assembler::vmovdqu(Address dst, XMMRegister src) {
  2327 void Assembler::vmovdqu(Address dst, XMMRegister src) {
  2311   _instruction_uses_vl = true;
       
  2312   assert(UseAVX > 0, "");
  2328   assert(UseAVX > 0, "");
  2313   if (VM_Version::supports_evex()) {
  2329   InstructionMark im(this);
  2314     _tuple_type = EVEX_FVM;
  2330   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  2315   }
  2331   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
  2316   InstructionMark im(this);
       
  2317   int vector_len = AVX_256bit;
       
  2318   // swap src<->dst for encoding
  2332   // swap src<->dst for encoding
  2319   assert(src != xnoreg, "sanity");
  2333   assert(src != xnoreg, "sanity");
  2320   vex_prefix(src, xnoreg, dst, VEX_SIMD_F3, vector_len);
  2334   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  2321   emit_int8(0x7F);
  2335   emit_int8(0x7F);
  2322   emit_operand(src, dst);
  2336   emit_operand(src, dst);
  2323 }
  2337 }
  2324 
  2338 
  2325 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
  2339 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
  2326 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
  2340 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
  2327   _instruction_uses_vl = true;
  2341   assert(VM_Version::supports_evex(), "");
  2328   assert(UseAVX > 0, "");
  2342   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  2329   int src_enc = src->encoding();
  2343   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  2330   int dst_enc = dst->encoding();
       
  2331   int encode = vex_prefix_and_encode(dst_enc, 0, src_enc, VEX_SIMD_F3, VEX_OPCODE_0F,
       
  2332                                      /* vex_w */ false, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
       
  2333   emit_int8(0x6F);
  2344   emit_int8(0x6F);
  2334   emit_int8((unsigned char)(0xC0 | encode));
  2345   emit_int8((unsigned char)(0xC0 | encode));
  2335 }
  2346 }
  2336 
  2347 
  2337 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
  2348 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
  2338   _instruction_uses_vl = true;
  2349   assert(VM_Version::supports_evex(), "");
  2339   assert(UseAVX > 0, "");
  2350   InstructionMark im(this);
  2340   InstructionMark im(this);
  2351   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
  2341   if (VM_Version::supports_evex()) {
  2352   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
  2342     _tuple_type = EVEX_FVM;
  2353   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  2343   }
       
  2344   vex_prefix(dst, xnoreg, src, VEX_SIMD_F3, vector_len);
       
  2345   emit_int8(0x6F);
  2354   emit_int8(0x6F);
  2346   emit_operand(dst, src);
  2355   emit_operand(dst, src);
  2347 }
  2356 }
  2348 
  2357 
  2349 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
  2358 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
  2350   _instruction_uses_vl = true;
  2359   assert(VM_Version::supports_evex(), "");
  2351   assert(UseAVX > 0, "");
       
  2352   InstructionMark im(this);
       
  2353   assert(src != xnoreg, "sanity");
  2360   assert(src != xnoreg, "sanity");
  2354   if (VM_Version::supports_evex()) {
  2361   InstructionMark im(this);
  2355     _tuple_type = EVEX_FVM;
  2362   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  2356   }
  2363   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
  2357   // swap src<->dst for encoding
  2364   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  2358   vex_prefix(src, xnoreg, dst, VEX_SIMD_F3, vector_len);
       
  2359   emit_int8(0x7F);
  2365   emit_int8(0x7F);
  2360   emit_operand(src, dst);
  2366   emit_operand(src, dst);
  2361 }
  2367 }
  2362 
  2368 
  2363 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
  2369 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
  2364   _instruction_uses_vl = true;
  2370   assert(VM_Version::supports_evex(), "");
  2365   assert(UseAVX > 0, "");
  2371   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  2366   int src_enc = src->encoding();
  2372   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  2367   int dst_enc = dst->encoding();
       
  2368   int encode = vex_prefix_and_encode(dst_enc, 0, src_enc, VEX_SIMD_F3, VEX_OPCODE_0F,
       
  2369                                      /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
       
  2370   emit_int8(0x6F);
  2373   emit_int8(0x6F);
  2371   emit_int8((unsigned char)(0xC0 | encode));
  2374   emit_int8((unsigned char)(0xC0 | encode));
  2372 }
  2375 }
  2373 
  2376 
  2374 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
  2377 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
  2375   _instruction_uses_vl = true;
  2378   assert(VM_Version::supports_evex(), "");
  2376   assert(UseAVX > 2, "");
  2379   InstructionMark im(this);
  2377   InstructionMark im(this);
  2380   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
  2378   _tuple_type = EVEX_FVM;
  2381   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
  2379   vex_prefix_q(dst, xnoreg, src, VEX_SIMD_F3, vector_len);
  2382   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  2380   emit_int8(0x6F);
  2383   emit_int8(0x6F);
  2381   emit_operand(dst, src);
  2384   emit_operand(dst, src);
  2382 }
  2385 }
  2383 
  2386 
  2384 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
  2387 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
  2385   _instruction_uses_vl = true;
  2388   assert(VM_Version::supports_evex(), "");
  2386   assert(UseAVX > 2, "");
       
  2387   InstructionMark im(this);
       
  2388   assert(src != xnoreg, "sanity");
  2389   assert(src != xnoreg, "sanity");
  2389   _tuple_type = EVEX_FVM;
  2390   InstructionMark im(this);
  2390   // swap src<->dst for encoding
  2391   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  2391   vex_prefix_q(src, xnoreg, dst, VEX_SIMD_F3, vector_len);
  2392   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
       
  2393   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  2392   emit_int8(0x7F);
  2394   emit_int8(0x7F);
  2393   emit_operand(src, dst);
  2395   emit_operand(src, dst);
  2394 }
  2396 }
  2395 
  2397 
  2396 // Uses zero extension on 64bit
  2398 // Uses zero extension on 64bit
  2432 // New cpus require to use movsd and movss to avoid partial register stall
  2434 // New cpus require to use movsd and movss to avoid partial register stall
  2433 // when loading from memory. But for old Opteron use movlpd instead of movsd.
  2435 // when loading from memory. But for old Opteron use movlpd instead of movsd.
  2434 // The selection is done in MacroAssembler::movdbl() and movflt().
  2436 // The selection is done in MacroAssembler::movdbl() and movflt().
  2435 void Assembler::movlpd(XMMRegister dst, Address src) {
  2437 void Assembler::movlpd(XMMRegister dst, Address src) {
  2436   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2438   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2437   if (VM_Version::supports_evex()) {
  2439   InstructionMark im(this);
  2438     _tuple_type = EVEX_T1S;
  2440   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  2439     _input_size_in_bits = EVEX_32bit;
  2441   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
  2440     emit_simd_arith_q(0x12, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
  2442   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  2441   } else {
  2443   emit_int8(0x12);
  2442     emit_simd_arith(0x12, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
  2444   emit_operand(dst, src);
  2443   }
       
  2444 }
  2445 }
  2445 
  2446 
  2446 void Assembler::movq( MMXRegister dst, Address src ) {
  2447 void Assembler::movq( MMXRegister dst, Address src ) {
  2447   assert( VM_Version::supports_mmx(), "" );
  2448   assert( VM_Version::supports_mmx(), "" );
  2448   emit_int8(0x0F);
  2449   emit_int8(0x0F);
  2464 }
  2465 }
  2465 
  2466 
  2466 void Assembler::movq(XMMRegister dst, Address src) {
  2467 void Assembler::movq(XMMRegister dst, Address src) {
  2467   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2468   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2468   InstructionMark im(this);
  2469   InstructionMark im(this);
  2469   if (VM_Version::supports_evex()) {
  2470   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  2470     _tuple_type = EVEX_T1S;
  2471   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  2471     _input_size_in_bits = EVEX_64bit;
  2472   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  2472     simd_prefix_q(dst, xnoreg, src, VEX_SIMD_F3, /* no_mask_reg */ true);
       
  2473   } else {
       
  2474     simd_prefix(dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
       
  2475   }
       
  2476   emit_int8(0x7E);
  2473   emit_int8(0x7E);
  2477   emit_operand(dst, src);
  2474   emit_operand(dst, src);
  2478 }
  2475 }
  2479 
  2476 
  2480 void Assembler::movq(Address dst, XMMRegister src) {
  2477 void Assembler::movq(Address dst, XMMRegister src) {
  2481   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2478   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2482   InstructionMark im(this);
  2479   InstructionMark im(this);
  2483   if (VM_Version::supports_evex()) {
  2480   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  2484     _tuple_type = EVEX_T1S;
  2481   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  2485     _input_size_in_bits = EVEX_64bit;
  2482   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  2486     simd_prefix(src, xnoreg, dst, VEX_SIMD_66, /* no_mask_reg */ true,
       
  2487                 VEX_OPCODE_0F, /* rex_w */ true);
       
  2488   } else {
       
  2489     simd_prefix(dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
       
  2490   }
       
  2491   emit_int8((unsigned char)0xD6);
  2483   emit_int8((unsigned char)0xD6);
  2492   emit_operand(src, dst);
  2484   emit_operand(src, dst);
  2493 }
  2485 }
  2494 
  2486 
  2495 void Assembler::movsbl(Register dst, Address src) { // movsxb
  2487 void Assembler::movsbl(Register dst, Address src) { // movsxb
  2508   emit_int8((unsigned char)(0xC0 | encode));
  2500   emit_int8((unsigned char)(0xC0 | encode));
  2509 }
  2501 }
  2510 
  2502 
  2511 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
  2503 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
  2512   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2504   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2513   if (VM_Version::supports_evex()) {
  2505   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  2514     emit_simd_arith_q(0x10, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);
  2506   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  2515   } else {
  2507   emit_int8(0x10);
  2516     emit_simd_arith(0x10, dst, src, VEX_SIMD_F2);
  2508   emit_int8((unsigned char)(0xC0 | encode));
  2517   }
       
  2518 }
  2509 }
  2519 
  2510 
  2520 void Assembler::movsd(XMMRegister dst, Address src) {
  2511 void Assembler::movsd(XMMRegister dst, Address src) {
  2521   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2512   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2522   if (VM_Version::supports_evex()) {
  2513   InstructionMark im(this);
  2523     _tuple_type = EVEX_T1S;
  2514   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  2524     _input_size_in_bits = EVEX_64bit;
  2515   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  2525     emit_simd_arith_nonds_q(0x10, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);
  2516   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  2526   } else {
  2517   emit_int8(0x10);
  2527     emit_simd_arith_nonds(0x10, dst, src, VEX_SIMD_F2);
  2518   emit_operand(dst, src);
  2528   }
       
  2529 }
  2519 }
  2530 
  2520 
  2531 void Assembler::movsd(Address dst, XMMRegister src) {
  2521 void Assembler::movsd(Address dst, XMMRegister src) {
  2532   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2522   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2533   InstructionMark im(this);
  2523   InstructionMark im(this);
  2534   if (VM_Version::supports_evex()) {
  2524   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  2535     _tuple_type = EVEX_T1S;
  2525   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  2536     _input_size_in_bits = EVEX_64bit;
  2526   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  2537     simd_prefix_q(src, xnoreg, dst, VEX_SIMD_F2);
       
  2538   } else {
       
  2539     simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, /* no_mask_reg */ false);
       
  2540   }
       
  2541   emit_int8(0x11);
  2527   emit_int8(0x11);
  2542   emit_operand(src, dst);
  2528   emit_operand(src, dst);
  2543 }
  2529 }
  2544 
  2530 
  2545 void Assembler::movss(XMMRegister dst, XMMRegister src) {
  2531 void Assembler::movss(XMMRegister dst, XMMRegister src) {
  2546   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  2532   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  2547   emit_simd_arith(0x10, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
  2533   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  2534   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
       
  2535   emit_int8(0x10);
       
  2536   emit_int8((unsigned char)(0xC0 | encode));
  2548 }
  2537 }
  2549 
  2538 
  2550 void Assembler::movss(XMMRegister dst, Address src) {
  2539 void Assembler::movss(XMMRegister dst, Address src) {
  2551   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  2540   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  2552   if (VM_Version::supports_evex()) {
  2541   InstructionMark im(this);
  2553     _tuple_type = EVEX_T1S;
  2542   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  2554     _input_size_in_bits = EVEX_32bit;
  2543   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
  2555   }
  2544   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  2556   emit_simd_arith_nonds(0x10, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
  2545   emit_int8(0x10);
       
  2546   emit_operand(dst, src);
  2557 }
  2547 }
  2558 
  2548 
  2559 void Assembler::movss(Address dst, XMMRegister src) {
  2549 void Assembler::movss(Address dst, XMMRegister src) {
  2560   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  2550   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  2561   if (VM_Version::supports_evex()) {
  2551   InstructionMark im(this);
  2562     _tuple_type = EVEX_T1S;
  2552   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  2563     _input_size_in_bits = EVEX_32bit;
  2553   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
  2564   }
  2554   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  2565   InstructionMark im(this);
       
  2566   simd_prefix(dst, src, VEX_SIMD_F3, /* no_mask_reg */ false);
       
  2567   emit_int8(0x11);
  2555   emit_int8(0x11);
  2568   emit_operand(src, dst);
  2556   emit_operand(src, dst);
  2569 }
  2557 }
  2570 
  2558 
  2571 void Assembler::movswl(Register dst, Address src) { // movsxw
  2559 void Assembler::movswl(Register dst, Address src) { // movsxw
  2653   emit_int8((unsigned char)(0xE0 | encode));
  2641   emit_int8((unsigned char)(0xE0 | encode));
  2654 }
  2642 }
  2655 
  2643 
  2656 void Assembler::mulsd(XMMRegister dst, Address src) {
  2644 void Assembler::mulsd(XMMRegister dst, Address src) {
  2657   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2645   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2658   if (VM_Version::supports_evex()) {
  2646   InstructionMark im(this);
  2659     _tuple_type = EVEX_T1S;
  2647   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  2660     _input_size_in_bits = EVEX_64bit;
  2648   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  2661     emit_simd_arith_q(0x59, dst, src, VEX_SIMD_F2);
  2649   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  2662   } else {
  2650   emit_int8(0x59);
  2663     emit_simd_arith(0x59, dst, src, VEX_SIMD_F2);
  2651   emit_operand(dst, src);
  2664   }
       
  2665 }
  2652 }
  2666 
  2653 
  2667 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
  2654 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
  2668   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2655   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2669   if (VM_Version::supports_evex()) {
  2656   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  2670     emit_simd_arith_q(0x59, dst, src, VEX_SIMD_F2);
  2657   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  2671   } else {
  2658   emit_int8(0x59);
  2672     emit_simd_arith(0x59, dst, src, VEX_SIMD_F2);
  2659   emit_int8((unsigned char)(0xC0 | encode));
  2673   }
       
  2674 }
  2660 }
  2675 
  2661 
  2676 void Assembler::mulss(XMMRegister dst, Address src) {
  2662 void Assembler::mulss(XMMRegister dst, Address src) {
  2677   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  2663   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  2678   if (VM_Version::supports_evex()) {
  2664   InstructionMark im(this);
  2679     _tuple_type = EVEX_T1S;
  2665   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  2680     _input_size_in_bits = EVEX_32bit;
  2666   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
  2681   }
  2667   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  2682   emit_simd_arith(0x59, dst, src, VEX_SIMD_F3);
  2668   emit_int8(0x59);
       
  2669   emit_operand(dst, src);
  2683 }
  2670 }
  2684 
  2671 
  2685 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
  2672 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
  2686   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  2673   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  2687   emit_simd_arith(0x59, dst, src, VEX_SIMD_F3);
  2674   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  2675   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
       
  2676   emit_int8(0x59);
       
  2677   emit_int8((unsigned char)(0xC0 | encode));
  2688 }
  2678 }
  2689 
  2679 
  2690 void Assembler::negl(Register dst) {
  2680 void Assembler::negl(Register dst) {
  2691   int encode = prefix_and_encode(dst->encoding());
  2681   int encode = prefix_and_encode(dst->encoding());
  2692   emit_int8((unsigned char)0xF7);
  2682   emit_int8((unsigned char)0xF7);
  2983 }
  2973 }
  2984 
  2974 
  2985 void Assembler::packuswb(XMMRegister dst, Address src) {
  2975 void Assembler::packuswb(XMMRegister dst, Address src) {
  2986   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2976   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2987   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
  2977   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
  2988   if (VM_Version::supports_evex()) {
  2978   InstructionMark im(this);
  2989     _tuple_type = EVEX_FV;
  2979   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
  2990     _input_size_in_bits = EVEX_32bit;
  2980   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
  2991   }
  2981   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  2992   emit_simd_arith(0x67, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  2982   emit_int8(0x67);
       
  2983   emit_operand(dst, src);
  2993 }
  2984 }
  2994 
  2985 
  2995 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
  2986 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
  2996   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2987   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  2997   emit_simd_arith(0x67, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  2988   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
       
  2989   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  2990   emit_int8(0x67);
       
  2991   emit_int8((unsigned char)(0xC0 | encode));
  2998 }
  2992 }
  2999 
  2993 
  3000 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  2994 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  3001   assert(UseAVX > 0, "some form of AVX must be enabled");
  2995   assert(UseAVX > 0, "some form of AVX must be enabled");
  3002   emit_vex_arith(0x67, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  2996   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
       
  2997   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  2998   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  2999   emit_int8(0x67);
       
  3000   emit_int8((unsigned char)(0xC0 | encode));
  3003 }
  3001 }
  3004 
  3002 
  3005 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
  3003 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
  3006   _instruction_uses_vl = true;
       
  3007   assert(VM_Version::supports_avx2(), "");
  3004   assert(VM_Version::supports_avx2(), "");
  3008   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false,
  3005   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  3009                                       VEX_OPCODE_0F_3A, /* rex_w */ true, vector_len);
  3006   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  3010   emit_int8(0x00);
  3007   emit_int8(0x00);
  3011   emit_int8(0xC0 | encode);
  3008   emit_int8(0xC0 | encode);
  3012   emit_int8(imm8);
  3009   emit_int8(imm8);
  3013 }
  3010 }
  3014 
  3011 
  3018 }
  3015 }
  3019 
  3016 
  3020 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
  3017 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
  3021   assert(VM_Version::supports_sse4_2(), "");
  3018   assert(VM_Version::supports_sse4_2(), "");
  3022   InstructionMark im(this);
  3019   InstructionMark im(this);
  3023   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F_3A,
  3020   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  3024               /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
  3021   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  3025   emit_int8(0x61);
  3022   emit_int8(0x61);
  3026   emit_operand(dst, src);
  3023   emit_operand(dst, src);
  3027   emit_int8(imm8);
  3024   emit_int8(imm8);
  3028 }
  3025 }
  3029 
  3026 
  3030 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
  3027 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
  3031   assert(VM_Version::supports_sse4_2(), "");
  3028   assert(VM_Version::supports_sse4_2(), "");
  3032   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false,
  3029   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  3033                                       VEX_OPCODE_0F_3A, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
  3030   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  3034   emit_int8(0x61);
  3031   emit_int8(0x61);
  3035   emit_int8((unsigned char)(0xC0 | encode));
  3032   emit_int8((unsigned char)(0xC0 | encode));
  3036   emit_int8(imm8);
  3033   emit_int8(imm8);
  3037 }
  3034 }
  3038 
  3035 
  3039 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
  3036 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
  3040   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3037   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3041   emit_simd_arith(0x75, dst, src, VEX_SIMD_66,
  3038   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  3042                   false, (VM_Version::supports_avx512dq() == false));
  3039   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  3040   emit_int8(0x75);
       
  3041   emit_int8((unsigned char)(0xC0 | encode));
  3043 }
  3042 }
  3044 
  3043 
  3045 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  3044 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  3046   assert(UseAVX > 0, "some form of AVX must be enabled");
  3045   assert(VM_Version::supports_avx(), "");
  3047   emit_vex_arith(0x75, dst, nds, src, VEX_SIMD_66, vector_len,
  3046   assert(!VM_Version::supports_evex(), "");
  3048                  false, (VM_Version::supports_avx512dq() == false));
  3047   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  3048   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  3049   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  3050   emit_int8(0x75);
       
  3051   emit_int8((unsigned char)(0xC0 | encode));
  3049 }
  3052 }
  3050 
  3053 
  3051 void Assembler::pmovmskb(Register dst, XMMRegister src) {
  3054 void Assembler::pmovmskb(Register dst, XMMRegister src) {
  3052   assert(VM_Version::supports_sse2(), "");
  3055   assert(VM_Version::supports_sse2(), "");
  3053   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, true, VEX_OPCODE_0F,
  3056   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  3054                                       false, AVX_128bit, (VM_Version::supports_avx512dq() == false));
  3057   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  3055   emit_int8((unsigned char)0xD7);
  3058   emit_int8((unsigned char)0xD7);
  3056   emit_int8((unsigned char)(0xC0 | encode));
  3059   emit_int8((unsigned char)(0xC0 | encode));
  3057 }
  3060 }
  3058 
  3061 
  3059 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
  3062 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
  3060   assert(VM_Version::supports_avx2(), "");
  3063   assert(VM_Version::supports_avx2(), "");
  3061   int vector_len = AVX_256bit;
  3064   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  3062   int encode = vex_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66,
  3065   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  3063                                      vector_len, VEX_OPCODE_0F, true, false);
       
  3064   emit_int8((unsigned char)0xD7);
  3066   emit_int8((unsigned char)0xD7);
  3065   emit_int8((unsigned char)(0xC0 | encode));
  3067   emit_int8((unsigned char)(0xC0 | encode));
  3066 }
  3068 }
  3067 
  3069 
  3068 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
  3070 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
  3069   assert(VM_Version::supports_sse4_1(), "");
  3071   assert(VM_Version::supports_sse4_1(), "");
  3070   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ true,
  3072   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
  3071                                       VEX_OPCODE_0F_3A, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_dq);
  3073   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  3072   emit_int8(0x16);
  3074   emit_int8(0x16);
  3073   emit_int8((unsigned char)(0xC0 | encode));
  3075   emit_int8((unsigned char)(0xC0 | encode));
  3074   emit_int8(imm8);
  3076   emit_int8(imm8);
  3075 }
  3077 }
  3076 
  3078 
  3077 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
  3079 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
  3078   assert(VM_Version::supports_sse4_1(), "");
  3080   assert(VM_Version::supports_sse4_1(), "");
  3079   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, /* no_mask_reg */  true,
  3081   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
  3080                                       VEX_OPCODE_0F_3A, /* rex_w */ true, AVX_128bit, /* legacy_mode */ _legacy_mode_dq);
  3082   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  3081   emit_int8(0x16);
  3083   emit_int8(0x16);
  3082   emit_int8((unsigned char)(0xC0 | encode));
  3084   emit_int8((unsigned char)(0xC0 | encode));
  3083   emit_int8(imm8);
  3085   emit_int8(imm8);
  3084 }
  3086 }
  3085 
  3087 
  3086 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
  3088 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
  3087   assert(VM_Version::supports_sse2(), "");
  3089   assert(VM_Version::supports_sse2(), "");
  3088   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ true,
  3090   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
  3089                                       VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
  3091   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  3090   emit_int8((unsigned char)0xC5);
  3092   emit_int8((unsigned char)0xC5);
  3091   emit_int8((unsigned char)(0xC0 | encode));
  3093   emit_int8((unsigned char)(0xC0 | encode));
  3092   emit_int8(imm8);
  3094   emit_int8(imm8);
  3093 }
  3095 }
  3094 
  3096 
  3095 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
  3097 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
  3096   assert(VM_Version::supports_sse4_1(), "");
  3098   assert(VM_Version::supports_sse4_1(), "");
  3097   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, /* no_mask_reg */ true,
  3099   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
  3098                                       VEX_OPCODE_0F_3A, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_dq);
  3100   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  3099   emit_int8(0x22);
  3101   emit_int8(0x22);
  3100   emit_int8((unsigned char)(0xC0 | encode));
  3102   emit_int8((unsigned char)(0xC0 | encode));
  3101   emit_int8(imm8);
  3103   emit_int8(imm8);
  3102 }
  3104 }
  3103 
  3105 
  3104 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
  3106 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
  3105   assert(VM_Version::supports_sse4_1(), "");
  3107   assert(VM_Version::supports_sse4_1(), "");
  3106   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, /* no_mask_reg */ true,
  3108   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
  3107                                       VEX_OPCODE_0F_3A, /* rex_w */ true, AVX_128bit, /* legacy_mode */ _legacy_mode_dq);
  3109   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  3108   emit_int8(0x22);
  3110   emit_int8(0x22);
  3109   emit_int8((unsigned char)(0xC0 | encode));
  3111   emit_int8((unsigned char)(0xC0 | encode));
  3110   emit_int8(imm8);
  3112   emit_int8(imm8);
  3111 }
  3113 }
  3112 
  3114 
  3113 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
  3115 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
  3114   assert(VM_Version::supports_sse2(), "");
  3116   assert(VM_Version::supports_sse2(), "");
  3115   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, /* no_mask_reg */ true,
  3117   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
  3116                                       VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
  3118   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  3117   emit_int8((unsigned char)0xC4);
  3119   emit_int8((unsigned char)0xC4);
  3118   emit_int8((unsigned char)(0xC0 | encode));
  3120   emit_int8((unsigned char)(0xC0 | encode));
  3119   emit_int8(imm8);
  3121   emit_int8(imm8);
  3120 }
  3122 }
  3121 
  3123 
  3122 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
  3124 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
  3123   assert(VM_Version::supports_sse4_1(), "");
  3125   assert(VM_Version::supports_sse4_1(), "");
  3124   if (VM_Version::supports_evex()) {
  3126   InstructionMark im(this);
  3125     _tuple_type = EVEX_HVM;
  3127   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
  3126   }
  3128   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
  3127   InstructionMark im(this);
  3129   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  3128   simd_prefix(dst, src, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F_38);
       
  3129   emit_int8(0x30);
  3130   emit_int8(0x30);
  3130   emit_operand(dst, src);
  3131   emit_operand(dst, src);
  3131 }
  3132 }
  3132 
  3133 
  3133 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
  3134 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
  3134   assert(VM_Version::supports_sse4_1(), "");
  3135   assert(VM_Version::supports_sse4_1(), "");
  3135   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F_38);
  3136   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
       
  3137   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  3136   emit_int8(0x30);
  3138   emit_int8(0x30);
  3137   emit_int8((unsigned char)(0xC0 | encode));
  3139   emit_int8((unsigned char)(0xC0 | encode));
  3138 }
  3140 }
  3139 
  3141 
  3140 void Assembler::vpmovzxbw(XMMRegister dst, Address src) {
  3142 void Assembler::vpmovzxbw(XMMRegister dst, Address src) {
  3141   assert(VM_Version::supports_avx(), "");
  3143   assert(VM_Version::supports_avx(), "");
  3142   InstructionMark im(this);
  3144   InstructionMark im(this);
  3143   bool vector256 = true;
       
  3144   assert(dst != xnoreg, "sanity");
  3145   assert(dst != xnoreg, "sanity");
  3145   int dst_enc = dst->encoding();
  3146   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
  3146   vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, false, vector256);
  3147   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
       
  3148   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  3147   emit_int8(0x30);
  3149   emit_int8(0x30);
  3148   emit_operand(dst, src);
  3150   emit_operand(dst, src);
  3149 }
  3151 }
  3150 
  3152 
  3151 // generic
  3153 // generic
  3244   emit_int8(p);
  3246   emit_int8(p);
  3245 }
  3247 }
  3246 
  3248 
  3247 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
  3249 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
  3248   assert(VM_Version::supports_ssse3(), "");
  3250   assert(VM_Version::supports_ssse3(), "");
  3249   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
  3251   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
  3250                                       VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
  3252   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  3251   emit_int8(0x00);
  3253   emit_int8(0x00);
  3252   emit_int8((unsigned char)(0xC0 | encode));
  3254   emit_int8((unsigned char)(0xC0 | encode));
  3253 }
  3255 }
  3254 
  3256 
  3255 void Assembler::pshufb(XMMRegister dst, Address src) {
  3257 void Assembler::pshufb(XMMRegister dst, Address src) {
  3256   assert(VM_Version::supports_ssse3(), "");
  3258   assert(VM_Version::supports_ssse3(), "");
  3257   if (VM_Version::supports_evex()) {
  3259   InstructionMark im(this);
  3258     _tuple_type = EVEX_FVM;
  3260   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
  3259   }
  3261   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
  3260   InstructionMark im(this);
  3262   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  3261   simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
       
  3262               VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
       
  3263   emit_int8(0x00);
  3263   emit_int8(0x00);
  3264   emit_operand(dst, src);
  3264   emit_operand(dst, src);
  3265 }
  3265 }
  3266 
  3266 
  3267 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
  3267 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
  3268   _instruction_uses_vl = true;
       
  3269   assert(isByte(mode), "invalid value");
  3268   assert(isByte(mode), "invalid value");
  3270   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3269   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3271   emit_simd_arith_nonds(0x70, dst, src, VEX_SIMD_66);
  3270   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_128bit;
       
  3271   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  3272   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  3273   emit_int8(0x70);
       
  3274   emit_int8((unsigned char)(0xC0 | encode));
  3272   emit_int8(mode & 0xFF);
  3275   emit_int8(mode & 0xFF);
  3273 }
  3276 }
  3274 
  3277 
  3275 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
  3278 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
  3276   _instruction_uses_vl = true;
       
  3277   assert(isByte(mode), "invalid value");
  3279   assert(isByte(mode), "invalid value");
  3278   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3280   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3279   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
  3281   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
  3280   if (VM_Version::supports_evex()) {
  3282   InstructionMark im(this);
  3281     _tuple_type = EVEX_FV;
  3283   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  3282     _input_size_in_bits = EVEX_32bit;
  3284   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
  3283   }
  3285   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  3284   InstructionMark im(this);
       
  3285   simd_prefix(dst, src, VEX_SIMD_66, /* no_mask_reg */ false);
       
  3286   emit_int8(0x70);
  3286   emit_int8(0x70);
  3287   emit_operand(dst, src);
  3287   emit_operand(dst, src);
  3288   emit_int8(mode & 0xFF);
  3288   emit_int8(mode & 0xFF);
  3289 }
  3289 }
  3290 
  3290 
  3291 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
  3291 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
  3292   assert(isByte(mode), "invalid value");
  3292   assert(isByte(mode), "invalid value");
  3293   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3293   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3294   emit_simd_arith_nonds(0x70, dst, src, VEX_SIMD_F2, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  3294   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
       
  3295   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
       
  3296   emit_int8(0x70);
       
  3297   emit_int8((unsigned char)(0xC0 | encode));
  3295   emit_int8(mode & 0xFF);
  3298   emit_int8(mode & 0xFF);
  3296 }
  3299 }
  3297 
  3300 
  3298 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
  3301 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
  3299   assert(isByte(mode), "invalid value");
  3302   assert(isByte(mode), "invalid value");
  3300   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3303   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3301   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
  3304   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
  3302   if (VM_Version::supports_evex()) {
  3305   InstructionMark im(this);
  3303     _tuple_type = EVEX_FVM;
  3306   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
  3304   }
  3307   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
  3305   InstructionMark im(this);
  3308   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  3306   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, /* no_mask_reg */ false,
       
  3307               VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
       
  3308   emit_int8(0x70);
  3309   emit_int8(0x70);
  3309   emit_operand(dst, src);
  3310   emit_operand(dst, src);
  3310   emit_int8(mode & 0xFF);
  3311   emit_int8(mode & 0xFF);
  3311 }
  3312 }
  3312 
  3313 
  3313 void Assembler::psrldq(XMMRegister dst, int shift) {
  3314 void Assembler::psrldq(XMMRegister dst, int shift) {
  3314   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
  3315   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
  3315   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3316   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
       
  3317   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
  3316   // XMM3 is for /3 encoding: 66 0F 73 /3 ib
  3318   // XMM3 is for /3 encoding: 66 0F 73 /3 ib
  3317   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, /* no_mask_reg */ true,
  3319   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  3318                                       VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
       
  3319   emit_int8(0x73);
  3320   emit_int8(0x73);
  3320   emit_int8((unsigned char)(0xC0 | encode));
  3321   emit_int8((unsigned char)(0xC0 | encode));
  3321   emit_int8(shift);
  3322   emit_int8(shift);
  3322 }
  3323 }
  3323 
  3324 
  3324 void Assembler::pslldq(XMMRegister dst, int shift) {
  3325 void Assembler::pslldq(XMMRegister dst, int shift) {
  3325   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
  3326   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
  3326   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3327   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
       
  3328   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
  3327   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
  3329   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
  3328   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, /* no_mask_reg */ true,
  3330   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  3329                                       VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
       
  3330   emit_int8(0x73);
  3331   emit_int8(0x73);
  3331   emit_int8((unsigned char)(0xC0 | encode));
  3332   emit_int8((unsigned char)(0xC0 | encode));
  3332   emit_int8(shift);
  3333   emit_int8(shift);
  3333 }
  3334 }
  3334 
  3335 
  3335 void Assembler::ptest(XMMRegister dst, Address src) {
  3336 void Assembler::ptest(XMMRegister dst, Address src) {
  3336   assert(VM_Version::supports_sse4_1(), "");
  3337   assert(VM_Version::supports_sse4_1(), "");
  3337   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
  3338   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
  3338   InstructionMark im(this);
  3339   InstructionMark im(this);
  3339   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false,
  3340   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  3340               VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
  3341   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  3341   emit_int8(0x17);
  3342   emit_int8(0x17);
  3342   emit_operand(dst, src);
  3343   emit_operand(dst, src);
  3343 }
  3344 }
  3344 
  3345 
  3345 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
  3346 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
  3346   assert(VM_Version::supports_sse4_1(), "");
  3347   assert(VM_Version::supports_sse4_1(), "");
  3347   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false,
  3348   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  3348                                       VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
  3349   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  3349   emit_int8(0x17);
  3350   emit_int8(0x17);
  3350   emit_int8((unsigned char)(0xC0 | encode));
  3351   emit_int8((unsigned char)(0xC0 | encode));
  3351 }
  3352 }
  3352 
  3353 
  3353 void Assembler::vptest(XMMRegister dst, Address src) {
  3354 void Assembler::vptest(XMMRegister dst, Address src) {
  3354   assert(VM_Version::supports_avx(), "");
  3355   assert(VM_Version::supports_avx(), "");
  3355   InstructionMark im(this);
  3356   InstructionMark im(this);
  3356   int vector_len = AVX_256bit;
  3357   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  3357   assert(dst != xnoreg, "sanity");
  3358   assert(dst != xnoreg, "sanity");
  3358   int dst_enc = dst->encoding();
       
  3359   // swap src<->dst for encoding
  3359   // swap src<->dst for encoding
  3360   vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* rex_w */ false,
  3360   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  3361              vector_len, /* legacy_mode  */ true, /* no_mask_reg */ false);
       
  3362   emit_int8(0x17);
  3361   emit_int8(0x17);
  3363   emit_operand(dst, src);
  3362   emit_operand(dst, src);
  3364 }
  3363 }
  3365 
  3364 
  3366 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
  3365 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
  3367   assert(VM_Version::supports_avx(), "");
  3366   assert(VM_Version::supports_avx(), "");
  3368   int vector_len = AVX_256bit;
  3367   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  3369   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38, /* legacy_mode */ true);
  3368   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  3370   emit_int8(0x17);
  3369   emit_int8(0x17);
  3371   emit_int8((unsigned char)(0xC0 | encode));
  3370   emit_int8((unsigned char)(0xC0 | encode));
  3372 }
  3371 }
  3373 
  3372 
  3374 void Assembler::punpcklbw(XMMRegister dst, Address src) {
  3373 void Assembler::punpcklbw(XMMRegister dst, Address src) {
  3375   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3374   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3376   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
  3375   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
  3377   if (VM_Version::supports_evex()) {
  3376   InstructionMark im(this);
  3378     _tuple_type = EVEX_FVM;
  3377   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
  3379   }
  3378   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
  3380   emit_simd_arith(0x60, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_vlbw);
  3379   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  3380   emit_int8(0x60);
       
  3381   emit_operand(dst, src);
  3381 }
  3382 }
  3382 
  3383 
  3383 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
  3384 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
  3384   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3385   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3385   emit_simd_arith(0x60, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_vlbw);
  3386   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
       
  3387   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  3388   emit_int8(0x60);
       
  3389   emit_int8((unsigned char)(0xC0 | encode));
  3386 }
  3390 }
  3387 
  3391 
  3388 void Assembler::punpckldq(XMMRegister dst, Address src) {
  3392 void Assembler::punpckldq(XMMRegister dst, Address src) {
  3389   _instruction_uses_vl = true;
       
  3390   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3393   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3391   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
  3394   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
  3392   if (VM_Version::supports_evex()) {
  3395   InstructionMark im(this);
  3393     _tuple_type = EVEX_FV;
  3396   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  3394     _input_size_in_bits = EVEX_32bit;
  3397   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
  3395   }
  3398   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  3396   emit_simd_arith(0x62, dst, src, VEX_SIMD_66);
  3399   emit_int8(0x62);
       
  3400   emit_operand(dst, src);
  3397 }
  3401 }
  3398 
  3402 
  3399 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
  3403 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
  3400   _instruction_uses_vl = true;
  3404   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3401   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3405   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  3402   emit_simd_arith(0x62, dst, src, VEX_SIMD_66);
  3406   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  3407   emit_int8(0x62);
       
  3408   emit_int8((unsigned char)(0xC0 | encode));
  3403 }
  3409 }
  3404 
  3410 
  3405 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
  3411 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
  3406   _instruction_uses_vl = true;
  3412   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3407   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3413   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  3408   if (VM_Version::supports_evex()) {
  3414   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  3409     emit_simd_arith_q(0x6C, dst, src, VEX_SIMD_66);
  3415   emit_int8(0x6C);
  3410   } else {
  3416   emit_int8((unsigned char)(0xC0 | encode));
  3411     emit_simd_arith(0x6C, dst, src, VEX_SIMD_66);
       
  3412   }
       
  3413 }
  3417 }
  3414 
  3418 
  3415 void Assembler::push(int32_t imm32) {
  3419 void Assembler::push(int32_t imm32) {
  3416   // in 64bits we push 64bits onto the stack but only
  3420   // in 64bits we push 64bits onto the stack but only
  3417   // take a 32bit immediate
  3421   // take a 32bit immediate
  3452   }
  3456   }
  3453 }
  3457 }
  3454 
  3458 
  3455 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
  3459 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
  3456   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  3460   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  3457   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, /* no_mask_reg */ false, VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
  3461   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  3462   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
  3458   emit_int8(0x53);
  3463   emit_int8(0x53);
  3459   emit_int8(0xC0 | encode);
  3464   emit_int8((unsigned char)(0xC0 | encode));
  3460 }
  3465 }
  3461 
  3466 
  3462 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
  3467 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
  3463   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  3468   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  3464   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, /* no_mask_reg */ false, VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
  3469   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  3470   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  3465   emit_int8(0x53);
  3471   emit_int8(0x53);
  3466   emit_int8(0xC0 | encode);
  3472   emit_int8((unsigned char)(0xC0 | encode));
  3467 }
  3473 }
  3468 
  3474 
  3469 void Assembler::rdtsc() {
  3475 void Assembler::rdtsc() {
  3470   emit_int8((unsigned char)0x0F);
  3476   emit_int8((unsigned char)0x0F);
  3471   emit_int8((unsigned char)0x31);
  3477   emit_int8((unsigned char)0x31);
  3620   emit_int8((unsigned char)0xA5);
  3626   emit_int8((unsigned char)0xA5);
  3621 }
  3627 }
  3622 
  3628 
  3623 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
  3629 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
  3624   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3630   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3625   if (VM_Version::supports_evex()) {
  3631   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  3626     emit_simd_arith_q(0x51, dst, src, VEX_SIMD_F2);
  3632   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  3627   } else {
  3633   emit_int8(0x51);
  3628     emit_simd_arith(0x51, dst, src, VEX_SIMD_F2);
  3634   emit_int8((unsigned char)(0xC0 | encode));
  3629   }
       
  3630 }
  3635 }
  3631 
  3636 
  3632 void Assembler::sqrtsd(XMMRegister dst, Address src) {
  3637 void Assembler::sqrtsd(XMMRegister dst, Address src) {
  3633   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3638   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3634   if (VM_Version::supports_evex()) {
  3639   InstructionMark im(this);
  3635     _tuple_type = EVEX_T1S;
  3640   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  3636     _input_size_in_bits = EVEX_64bit;
  3641   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  3637     emit_simd_arith_q(0x51, dst, src, VEX_SIMD_F2);
  3642   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  3638   } else {
  3643   emit_int8(0x51);
  3639     emit_simd_arith(0x51, dst, src, VEX_SIMD_F2);
  3644   emit_operand(dst, src);
  3640   }
       
  3641 }
  3645 }
  3642 
  3646 
  3643 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
  3647 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
  3644   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  3648   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  3645   emit_simd_arith(0x51, dst, src, VEX_SIMD_F3);
  3649   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  3650   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
       
  3651   emit_int8(0x51);
       
  3652   emit_int8((unsigned char)(0xC0 | encode));
  3646 }
  3653 }
  3647 
  3654 
  3648 void Assembler::std() {
  3655 void Assembler::std() {
  3649   emit_int8((unsigned char)0xFD);
  3656   emit_int8((unsigned char)0xFD);
  3650 }
  3657 }
  3651 
  3658 
  3652 void Assembler::sqrtss(XMMRegister dst, Address src) {
  3659 void Assembler::sqrtss(XMMRegister dst, Address src) {
  3653   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  3660   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  3654   if (VM_Version::supports_evex()) {
  3661   InstructionMark im(this);
  3655     _tuple_type = EVEX_T1S;
  3662   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  3656     _input_size_in_bits = EVEX_32bit;
  3663   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
  3657   }
  3664   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  3658   emit_simd_arith(0x51, dst, src, VEX_SIMD_F3);
  3665   emit_int8(0x51);
       
  3666   emit_operand(dst, src);
  3659 }
  3667 }
  3660 
  3668 
  3661 void Assembler::stmxcsr( Address dst) {
  3669 void Assembler::stmxcsr( Address dst) {
  3662   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  3670   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  3663   InstructionMark im(this);
  3671   InstructionMark im(this);
  3703   emit_arith(0x2B, 0xC0, dst, src);
  3711   emit_arith(0x2B, 0xC0, dst, src);
  3704 }
  3712 }
  3705 
  3713 
  3706 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
  3714 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
  3707   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3715   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3708   if (VM_Version::supports_evex()) {
  3716   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  3709     emit_simd_arith_q(0x5C, dst, src, VEX_SIMD_F2);
  3717   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  3710   } else {
  3718   emit_int8(0x5C);
  3711     emit_simd_arith(0x5C, dst, src, VEX_SIMD_F2);
  3719   emit_int8((unsigned char)(0xC0 | encode));
  3712   }
       
  3713 }
  3720 }
  3714 
  3721 
  3715 void Assembler::subsd(XMMRegister dst, Address src) {
  3722 void Assembler::subsd(XMMRegister dst, Address src) {
  3716   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3723   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3717   if (VM_Version::supports_evex()) {
  3724   InstructionMark im(this);
  3718     _tuple_type = EVEX_T1S;
  3725   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  3719     _input_size_in_bits = EVEX_64bit;
  3726   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  3720   }
  3727   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  3721   if (VM_Version::supports_evex()) {
  3728   emit_int8(0x5C);
  3722     emit_simd_arith_q(0x5C, dst, src, VEX_SIMD_F2);
  3729   emit_operand(dst, src);
  3723   } else {
       
  3724     emit_simd_arith(0x5C, dst, src, VEX_SIMD_F2);
       
  3725   }
       
  3726 }
  3730 }
  3727 
  3731 
  3728 void Assembler::subss(XMMRegister dst, XMMRegister src) {
  3732 void Assembler::subss(XMMRegister dst, XMMRegister src) {
  3729   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  3733   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  3730   emit_simd_arith(0x5C, dst, src, VEX_SIMD_F3);
  3734   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  3735   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
       
  3736   emit_int8(0x5C);
       
  3737   emit_int8((unsigned char)(0xC0 | encode));
  3731 }
  3738 }
  3732 
  3739 
  3733 void Assembler::subss(XMMRegister dst, Address src) {
  3740 void Assembler::subss(XMMRegister dst, Address src) {
  3734   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  3741   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  3735   if (VM_Version::supports_evex()) {
  3742   InstructionMark im(this);
  3736     _tuple_type = EVEX_T1S;
  3743   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  3737     _input_size_in_bits = EVEX_32bit;
  3744   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
  3738   }
  3745   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  3739   emit_simd_arith(0x5C, dst, src, VEX_SIMD_F3);
  3746   emit_int8(0x5C);
       
  3747   emit_operand(dst, src);
  3740 }
  3748 }
  3741 
  3749 
  3742 void Assembler::testb(Register dst, int imm8) {
  3750 void Assembler::testb(Register dst, int imm8) {
  3743   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
  3751   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
  3744   (void) prefix_and_encode(dst->encoding(), true);
  3752   (void) prefix_and_encode(dst->encoding(), true);
  3763 void Assembler::testl(Register dst, Register src) {
  3771 void Assembler::testl(Register dst, Register src) {
  3764   (void) prefix_and_encode(dst->encoding(), src->encoding());
  3772   (void) prefix_and_encode(dst->encoding(), src->encoding());
  3765   emit_arith(0x85, 0xC0, dst, src);
  3773   emit_arith(0x85, 0xC0, dst, src);
  3766 }
  3774 }
  3767 
  3775 
  3768 void Assembler::testl(Register dst, Address  src) {
  3776 void Assembler::testl(Register dst, Address src) {
  3769   InstructionMark im(this);
  3777   InstructionMark im(this);
  3770   prefix(src, dst);
  3778   prefix(src, dst);
  3771   emit_int8((unsigned char)0x85);
  3779   emit_int8((unsigned char)0x85);
  3772   emit_operand(dst, src);
  3780   emit_operand(dst, src);
  3773 }
  3781 }
  3790   emit_int8((unsigned char)(0xC0 | encode));
  3798   emit_int8((unsigned char)(0xC0 | encode));
  3791 }
  3799 }
  3792 
  3800 
  3793 void Assembler::ucomisd(XMMRegister dst, Address src) {
  3801 void Assembler::ucomisd(XMMRegister dst, Address src) {
  3794   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3802   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3795   if (VM_Version::supports_evex()) {
  3803   InstructionMark im(this);
  3796     _tuple_type = EVEX_T1S;
  3804   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  3797     _input_size_in_bits = EVEX_64bit;
  3805   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  3798     emit_simd_arith_nonds_q(0x2E, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
  3806   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  3799   } else {
  3807   emit_int8(0x2E);
  3800     emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_66);
  3808   emit_operand(dst, src);
  3801   }
       
  3802 }
  3809 }
  3803 
  3810 
  3804 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
  3811 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
  3805   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3812   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  3806   if (VM_Version::supports_evex()) {
  3813   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  3807     emit_simd_arith_nonds_q(0x2E, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
  3814   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  3808   } else {
  3815   emit_int8(0x2E);
  3809     emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_66);
  3816   emit_int8((unsigned char)(0xC0 | encode));
  3810   }
       
  3811 }
  3817 }
  3812 
  3818 
  3813 void Assembler::ucomiss(XMMRegister dst, Address src) {
  3819 void Assembler::ucomiss(XMMRegister dst, Address src) {
  3814   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  3820   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  3815   if (VM_Version::supports_evex()) {
  3821   InstructionMark im(this);
  3816     _tuple_type = EVEX_T1S;
  3822   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  3817     _input_size_in_bits = EVEX_32bit;
  3823   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
  3818   }
  3824   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
  3819   emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ true);
  3825   emit_int8(0x2E);
       
  3826   emit_operand(dst, src);
  3820 }
  3827 }
  3821 
  3828 
  3822 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
  3829 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
  3823   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  3830   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  3824   emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ true);
  3831   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  3832   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  3833   emit_int8(0x2E);
       
  3834   emit_int8((unsigned char)(0xC0 | encode));
  3825 }
  3835 }
  3826 
  3836 
  3827 void Assembler::xabort(int8_t imm8) {
  3837 void Assembler::xabort(int8_t imm8) {
  3828   emit_int8((unsigned char)0xC6);
  3838   emit_int8((unsigned char)0xC6);
  3829   emit_int8((unsigned char)0xF8);
  3839   emit_int8((unsigned char)0xF8);
  3901 
  3911 
  3902 // AVX 3-operands scalar float-point arithmetic instructions
  3912 // AVX 3-operands scalar float-point arithmetic instructions
  3903 
  3913 
  3904 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
  3914 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
  3905   assert(VM_Version::supports_avx(), "");
  3915   assert(VM_Version::supports_avx(), "");
  3906   if (VM_Version::supports_evex()) {
  3916   InstructionMark im(this);
  3907     _tuple_type = EVEX_T1S;
  3917   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  3908     _input_size_in_bits = EVEX_64bit;
  3918   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  3909     emit_vex_arith_q(0x58, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
  3919   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  3910   } else {
  3920   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  3911     emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
  3921   emit_int8(0x58);
  3912   }
  3922   emit_operand(dst, src);
  3913 }
  3923 }
  3914 
  3924 
  3915 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  3925 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  3916   assert(VM_Version::supports_avx(), "");
  3926   assert(VM_Version::supports_avx(), "");
  3917   if (VM_Version::supports_evex()) {
  3927   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  3918     emit_vex_arith_q(0x58, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
  3928   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  3919   } else {
  3929   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  3920     emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
  3930   emit_int8(0x58);
  3921   }
  3931   emit_int8((unsigned char)(0xC0 | encode));
  3922 }
  3932 }
  3923 
  3933 
  3924 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
  3934 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
  3925   assert(VM_Version::supports_avx(), "");
  3935   assert(VM_Version::supports_avx(), "");
  3926   if (VM_Version::supports_evex()) {
  3936   InstructionMark im(this);
  3927     _tuple_type = EVEX_T1S;
  3937   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  3928     _input_size_in_bits = EVEX_32bit;
  3938   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
  3929   }
  3939   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  3930   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
  3940   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
       
  3941   emit_int8(0x58);
       
  3942   emit_operand(dst, src);
  3931 }
  3943 }
  3932 
  3944 
  3933 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  3945 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  3934   assert(VM_Version::supports_avx(), "");
  3946   assert(VM_Version::supports_avx(), "");
  3935   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
  3947   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  3948   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  3949   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
       
  3950   emit_int8(0x58);
       
  3951   emit_int8((unsigned char)(0xC0 | encode));
  3936 }
  3952 }
  3937 
  3953 
  3938 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
  3954 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
  3939   assert(VM_Version::supports_avx(), "");
  3955   assert(VM_Version::supports_avx(), "");
  3940   if (VM_Version::supports_evex()) {
  3956   InstructionMark im(this);
  3941     _tuple_type = EVEX_T1S;
  3957   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  3942     _input_size_in_bits = EVEX_64bit;
  3958   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  3943     emit_vex_arith_q(0x5E, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
  3959   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  3944   } else {
  3960   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  3945     emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
  3961   emit_int8(0x5E);
  3946   }
  3962   emit_operand(dst, src);
  3947 }
  3963 }
  3948 
  3964 
  3949 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  3965 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  3950   assert(VM_Version::supports_avx(), "");
  3966   assert(VM_Version::supports_avx(), "");
  3951   if (VM_Version::supports_evex()) {
  3967   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  3952     emit_vex_arith_q(0x5E, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
  3968   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  3953   } else {
  3969   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  3954     emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
  3970   emit_int8(0x5E);
  3955   }
  3971   emit_int8((unsigned char)(0xC0 | encode));
  3956 }
  3972 }
  3957 
  3973 
  3958 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
  3974 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
  3959   assert(VM_Version::supports_avx(), "");
  3975   assert(VM_Version::supports_avx(), "");
  3960   if (VM_Version::supports_evex()) {
  3976   InstructionMark im(this);
  3961     _tuple_type = EVEX_T1S;
  3977   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  3962     _input_size_in_bits = EVEX_32bit;
  3978   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
  3963   }
  3979   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  3964   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
  3980   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
       
  3981   emit_int8(0x5E);
       
  3982   emit_operand(dst, src);
  3965 }
  3983 }
  3966 
  3984 
  3967 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  3985 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  3968   assert(VM_Version::supports_avx(), "");
  3986   assert(VM_Version::supports_avx(), "");
  3969   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
  3987   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  3988   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  3989   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
       
  3990   emit_int8(0x5E);
       
  3991   emit_int8((unsigned char)(0xC0 | encode));
  3970 }
  3992 }
  3971 
  3993 
  3972 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
  3994 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
  3973   assert(VM_Version::supports_avx(), "");
  3995   assert(VM_Version::supports_avx(), "");
  3974   if (VM_Version::supports_evex()) {
  3996   InstructionMark im(this);
  3975     _tuple_type = EVEX_T1S;
  3997   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  3976     _input_size_in_bits = EVEX_64bit;
  3998   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  3977     emit_vex_arith_q(0x59, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
  3999   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  3978   } else {
  4000   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  3979     emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
  4001   emit_int8(0x59);
  3980   }
  4002   emit_operand(dst, src);
  3981 }
  4003 }
  3982 
  4004 
  3983 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  4005 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  3984   assert(VM_Version::supports_avx(), "");
  4006   assert(VM_Version::supports_avx(), "");
  3985   if (VM_Version::supports_evex()) {
  4007   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  3986     emit_vex_arith_q(0x59, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
  4008   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  3987   } else {
  4009   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  3988     emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
  4010   emit_int8(0x59);
  3989   }
  4011   emit_int8((unsigned char)(0xC0 | encode));
  3990 }
  4012 }
  3991 
  4013 
  3992 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
  4014 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
  3993   assert(VM_Version::supports_avx(), "");
  4015   assert(VM_Version::supports_avx(), "");
  3994   if (VM_Version::supports_evex()) {
  4016   InstructionMark im(this);
  3995     _tuple_type = EVEX_T1S;
  4017   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  3996     _input_size_in_bits = EVEX_32bit;
  4018   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
  3997   }
  4019   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  3998   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
  4020   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
       
  4021   emit_int8(0x59);
       
  4022   emit_operand(dst, src);
  3999 }
  4023 }
  4000 
  4024 
  4001 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  4025 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  4002   assert(VM_Version::supports_avx(), "");
  4026   assert(VM_Version::supports_avx(), "");
  4003   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
  4027   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  4028   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4029   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
       
  4030   emit_int8(0x59);
       
  4031   emit_int8((unsigned char)(0xC0 | encode));
  4004 }
  4032 }
  4005 
  4033 
  4006 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
  4034 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
  4007   assert(VM_Version::supports_avx(), "");
  4035   assert(VM_Version::supports_avx(), "");
  4008   if (VM_Version::supports_evex()) {
  4036   InstructionMark im(this);
  4009     _tuple_type = EVEX_T1S;
  4037   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  4010     _input_size_in_bits = EVEX_64bit;
  4038   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  4011     emit_vex_arith_q(0x5C, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
  4039   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4012   } else {
  4040   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  4013     emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
  4041   emit_int8(0x5C);
  4014   }
  4042   emit_operand(dst, src);
  4015 }
  4043 }
  4016 
  4044 
  4017 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  4045 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  4018   assert(VM_Version::supports_avx(), "");
  4046   assert(VM_Version::supports_avx(), "");
  4019   if (VM_Version::supports_evex()) {
  4047   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  4020     emit_vex_arith_q(0x5C, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
  4048   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4021   } else {
  4049   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  4022     emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
  4050   emit_int8(0x5C);
  4023   }
  4051   emit_int8((unsigned char)(0xC0 | encode));
  4024 }
  4052 }
  4025 
  4053 
  4026 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
  4054 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
  4027   assert(VM_Version::supports_avx(), "");
  4055   assert(VM_Version::supports_avx(), "");
  4028   if (VM_Version::supports_evex()) {
  4056   InstructionMark im(this);
  4029     _tuple_type = EVEX_T1S;
  4057   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  4030     _input_size_in_bits = EVEX_32bit;
  4058   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
  4031   }
  4059   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4032   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
  4060   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
       
  4061   emit_int8(0x5C);
       
  4062   emit_operand(dst, src);
  4033 }
  4063 }
  4034 
  4064 
  4035 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  4065 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  4036   assert(VM_Version::supports_avx(), "");
  4066   assert(VM_Version::supports_avx(), "");
  4037   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
  4067   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  4068   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4069   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
       
  4070   emit_int8(0x5C);
       
  4071   emit_int8((unsigned char)(0xC0 | encode));
  4038 }
  4072 }
  4039 
  4073 
  4040 //====================VECTOR ARITHMETIC=====================================
  4074 //====================VECTOR ARITHMETIC=====================================
  4041 
  4075 
  4042 // Float-point vector arithmetic
  4076 // Float-point vector arithmetic
  4043 
  4077 
  4044 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
  4078 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
  4045   _instruction_uses_vl = true;
  4079   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4046   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4080   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4047   if (VM_Version::supports_evex()) {
  4081   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4048     emit_simd_arith_q(0x58, dst, src, VEX_SIMD_66);
  4082   emit_int8(0x58);
  4049   } else {
  4083   emit_int8((unsigned char)(0xC0 | encode));
  4050     emit_simd_arith(0x58, dst, src, VEX_SIMD_66);
       
  4051   }
       
  4052 }
  4084 }
  4053 
  4085 
  4054 void Assembler::addps(XMMRegister dst, XMMRegister src) {
  4086 void Assembler::addps(XMMRegister dst, XMMRegister src) {
  4055   _instruction_uses_vl = true;
  4087   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4056   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4088   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4057   emit_simd_arith(0x58, dst, src, VEX_SIMD_NONE);
  4089   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  4090   emit_int8(0x58);
       
  4091   emit_int8((unsigned char)(0xC0 | encode));
  4058 }
  4092 }
  4059 
  4093 
  4060 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4094 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4061   _instruction_uses_vl = true;
       
  4062   assert(VM_Version::supports_avx(), "");
  4095   assert(VM_Version::supports_avx(), "");
  4063   if (VM_Version::supports_evex()) {
  4096   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4064     emit_vex_arith_q(0x58, dst, nds, src, VEX_SIMD_66, vector_len);
  4097   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4065   } else {
  4098   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4066     emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_66, vector_len);
  4099   emit_int8(0x58);
  4067   }
  4100   emit_int8((unsigned char)(0xC0 | encode));
  4068 }
  4101 }
  4069 
  4102 
  4070 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4103 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4071   _instruction_uses_vl = true;
       
  4072   assert(VM_Version::supports_avx(), "");
  4104   assert(VM_Version::supports_avx(), "");
  4073   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_NONE, vector_len);
  4105   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4106   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4107   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  4108   emit_int8(0x58);
       
  4109   emit_int8((unsigned char)(0xC0 | encode));
  4074 }
  4110 }
  4075 
  4111 
  4076 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4112 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4077   _instruction_uses_vl = true;
       
  4078   assert(VM_Version::supports_avx(), "");
  4113   assert(VM_Version::supports_avx(), "");
  4079   if (VM_Version::supports_evex()) {
  4114   InstructionMark im(this);
  4080     _tuple_type = EVEX_FV;
  4115   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4081     _input_size_in_bits = EVEX_64bit;
  4116   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
  4082     emit_vex_arith_q(0x58, dst, nds, src, VEX_SIMD_66, vector_len);
  4117   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4083   } else {
  4118   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4084     emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_66, vector_len);
  4119   emit_int8(0x58);
  4085   }
  4120   emit_operand(dst, src);
  4086 }
  4121 }
  4087 
  4122 
  4088 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4123 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4089   _instruction_uses_vl = true;
       
  4090   assert(VM_Version::supports_avx(), "");
  4124   assert(VM_Version::supports_avx(), "");
  4091   if (VM_Version::supports_evex()) {
  4125   InstructionMark im(this);
  4092     _tuple_type = EVEX_FV;
  4126   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4093     _input_size_in_bits = EVEX_32bit;
  4127   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
  4094   }
  4128   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4095   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_NONE, vector_len);
  4129   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  4130   emit_int8(0x58);
       
  4131   emit_operand(dst, src);
  4096 }
  4132 }
  4097 
  4133 
  4098 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
  4134 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
  4099   _instruction_uses_vl = true;
  4135   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4100   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4136   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4101   if (VM_Version::supports_evex()) {
  4137   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4102     emit_simd_arith_q(0x5C, dst, src, VEX_SIMD_66);
  4138   emit_int8(0x5C);
  4103   } else {
  4139   emit_int8((unsigned char)(0xC0 | encode));
  4104     emit_simd_arith(0x5C, dst, src, VEX_SIMD_66);
       
  4105   }
       
  4106 }
  4140 }
  4107 
  4141 
  4108 void Assembler::subps(XMMRegister dst, XMMRegister src) {
  4142 void Assembler::subps(XMMRegister dst, XMMRegister src) {
  4109   _instruction_uses_vl = true;
  4143   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4110   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4144   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4111   emit_simd_arith(0x5C, dst, src, VEX_SIMD_NONE);
  4145   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  4146   emit_int8(0x5C);
       
  4147   emit_int8((unsigned char)(0xC0 | encode));
  4112 }
  4148 }
  4113 
  4149 
  4114 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4150 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4115   _instruction_uses_vl = true;
       
  4116   assert(VM_Version::supports_avx(), "");
  4151   assert(VM_Version::supports_avx(), "");
  4117   if (VM_Version::supports_evex()) {
  4152   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4118     emit_vex_arith_q(0x5C, dst, nds, src, VEX_SIMD_66, vector_len);
  4153   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4119   } else {
  4154   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4120     emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_66, vector_len);
  4155   emit_int8(0x5C);
  4121   }
  4156   emit_int8((unsigned char)(0xC0 | encode));
  4122 }
  4157 }
  4123 
  4158 
  4124 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4159 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4125   _instruction_uses_vl = true;
       
  4126   assert(VM_Version::supports_avx(), "");
  4160   assert(VM_Version::supports_avx(), "");
  4127   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_NONE, vector_len);
  4161   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4162   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4163   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  4164   emit_int8(0x5C);
       
  4165   emit_int8((unsigned char)(0xC0 | encode));
  4128 }
  4166 }
  4129 
  4167 
  4130 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4168 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4131   _instruction_uses_vl = true;
       
  4132   assert(VM_Version::supports_avx(), "");
  4169   assert(VM_Version::supports_avx(), "");
  4133   if (VM_Version::supports_evex()) {
  4170   InstructionMark im(this);
  4134     _tuple_type = EVEX_FV;
  4171   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4135     _input_size_in_bits = EVEX_64bit;
  4172   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
  4136     emit_vex_arith_q(0x5C, dst, nds, src, VEX_SIMD_66, vector_len);
  4173   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4137   } else {
  4174   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4138     emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_66, vector_len);
  4175   emit_int8(0x5C);
  4139   }
  4176   emit_operand(dst, src);
  4140 }
  4177 }
  4141 
  4178 
  4142 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4179 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4143   _instruction_uses_vl = true;
       
  4144   assert(VM_Version::supports_avx(), "");
  4180   assert(VM_Version::supports_avx(), "");
  4145   if (VM_Version::supports_evex()) {
  4181   InstructionMark im(this);
  4146     _tuple_type = EVEX_FV;
  4182   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4147     _input_size_in_bits = EVEX_32bit;
  4183   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
  4148   }
  4184   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4149   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_NONE, vector_len);
  4185   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  4186   emit_int8(0x5C);
       
  4187   emit_operand(dst, src);
  4150 }
  4188 }
  4151 
  4189 
  4152 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
  4190 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
  4153   _instruction_uses_vl = true;
  4191   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4154   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4192   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4155   if (VM_Version::supports_evex()) {
  4193   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4156     emit_simd_arith_q(0x59, dst, src, VEX_SIMD_66);
  4194   emit_int8(0x59);
  4157   } else {
  4195   emit_int8((unsigned char)(0xC0 | encode));
  4158     emit_simd_arith(0x59, dst, src, VEX_SIMD_66);
       
  4159   }
       
  4160 }
  4196 }
  4161 
  4197 
  4162 void Assembler::mulpd(XMMRegister dst, Address src) {
  4198 void Assembler::mulpd(XMMRegister dst, Address src) {
  4163   _instruction_uses_vl = true;
  4199   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4164   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4200   InstructionMark im(this);
  4165   if (VM_Version::supports_evex()) {
  4201   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4166     emit_simd_arith_q(0x59, dst, src, VEX_SIMD_66);
  4202   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
  4167   } else {
  4203   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4168     emit_simd_arith(0x59, dst, src, VEX_SIMD_66);
  4204   emit_int8(0x59);
  4169   }
  4205   emit_operand(dst, src);
  4170 }
  4206 }
  4171 
  4207 
  4172 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
  4208 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
  4173   _instruction_uses_vl = true;
  4209   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4174   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4210   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4175   emit_simd_arith(0x59, dst, src, VEX_SIMD_NONE);
  4211   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  4212   emit_int8(0x59);
       
  4213   emit_int8((unsigned char)(0xC0 | encode));
  4176 }
  4214 }
  4177 
  4215 
  4178 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4216 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4179   _instruction_uses_vl = true;
       
  4180   assert(VM_Version::supports_avx(), "");
  4217   assert(VM_Version::supports_avx(), "");
  4181   if (VM_Version::supports_evex()) {
  4218   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4182     emit_vex_arith_q(0x59, dst, nds, src, VEX_SIMD_66, vector_len);
  4219   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4183   } else {
  4220   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4184     emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_66, vector_len);
  4221   emit_int8(0x59);
  4185   }
  4222   emit_int8((unsigned char)(0xC0 | encode));
  4186 }
  4223 }
  4187 
  4224 
  4188 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4225 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4189   _instruction_uses_vl = true;
       
  4190   assert(VM_Version::supports_avx(), "");
  4226   assert(VM_Version::supports_avx(), "");
  4191   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_NONE, vector_len);
  4227   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4228   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4229   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  4230   emit_int8(0x59);
       
  4231   emit_int8((unsigned char)(0xC0 | encode));
  4192 }
  4232 }
  4193 
  4233 
  4194 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4234 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4195   _instruction_uses_vl = true;
       
  4196   assert(VM_Version::supports_avx(), "");
  4235   assert(VM_Version::supports_avx(), "");
  4197   if (VM_Version::supports_evex()) {
  4236   InstructionMark im(this);
  4198     _tuple_type = EVEX_FV;
  4237   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4199     _input_size_in_bits = EVEX_64bit;
  4238   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
  4200     emit_vex_arith_q(0x59, dst, nds, src, VEX_SIMD_66, vector_len);
  4239   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4201   } else {
  4240   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4202     emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_66, vector_len);
  4241   emit_int8(0x59);
  4203   }
  4242   emit_operand(dst, src);
  4204 }
  4243 }
  4205 
  4244 
  4206 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4245 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4207   _instruction_uses_vl = true;
       
  4208   assert(VM_Version::supports_avx(), "");
  4246   assert(VM_Version::supports_avx(), "");
  4209   if (VM_Version::supports_evex()) {
  4247   InstructionMark im(this);
  4210     _tuple_type = EVEX_FV;
  4248   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4211     _input_size_in_bits = EVEX_32bit;
  4249   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
  4212   }
  4250   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4213   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_NONE, vector_len);
  4251   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  4252   emit_int8(0x59);
       
  4253   emit_operand(dst, src);
  4214 }
  4254 }
  4215 
  4255 
  4216 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
  4256 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
  4217   _instruction_uses_vl = true;
  4257   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4218   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4258   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4219   if (VM_Version::supports_evex()) {
  4259   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4220     emit_simd_arith_q(0x5E, dst, src, VEX_SIMD_66);
  4260   emit_int8(0x5E);
  4221   } else {
  4261   emit_int8((unsigned char)(0xC0 | encode));
  4222     emit_simd_arith(0x5E, dst, src, VEX_SIMD_66);
       
  4223   }
       
  4224 }
  4262 }
  4225 
  4263 
  4226 void Assembler::divps(XMMRegister dst, XMMRegister src) {
  4264 void Assembler::divps(XMMRegister dst, XMMRegister src) {
  4227   _instruction_uses_vl = true;
  4265   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4228   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4266   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4229   emit_simd_arith(0x5E, dst, src, VEX_SIMD_NONE);
  4267   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  4268   emit_int8(0x5E);
       
  4269   emit_int8((unsigned char)(0xC0 | encode));
  4230 }
  4270 }
  4231 
  4271 
  4232 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4272 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4233   _instruction_uses_vl = true;
       
  4234   assert(VM_Version::supports_avx(), "");
  4273   assert(VM_Version::supports_avx(), "");
  4235   if (VM_Version::supports_evex()) {
  4274   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4236     emit_vex_arith_q(0x5E, dst, nds, src, VEX_SIMD_66, vector_len);
  4275   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4237   } else {
  4276   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4238     emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_66, vector_len);
  4277   emit_int8(0x5E);
  4239   }
  4278   emit_int8((unsigned char)(0xC0 | encode));
  4240 }
  4279 }
  4241 
  4280 
  4242 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4281 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4243   _instruction_uses_vl = true;
       
  4244   assert(VM_Version::supports_avx(), "");
  4282   assert(VM_Version::supports_avx(), "");
  4245   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_NONE, vector_len);
  4283   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4284   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4285   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  4286   emit_int8(0x5E);
       
  4287   emit_int8((unsigned char)(0xC0 | encode));
  4246 }
  4288 }
  4247 
  4289 
  4248 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4290 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4249   _instruction_uses_vl = true;
       
  4250   assert(VM_Version::supports_avx(), "");
  4291   assert(VM_Version::supports_avx(), "");
  4251   if (VM_Version::supports_evex()) {
  4292   InstructionMark im(this);
  4252     _tuple_type = EVEX_FV;
  4293   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4253     _input_size_in_bits = EVEX_64bit;
  4294   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
  4254     emit_vex_arith_q(0x5E, dst, nds, src, VEX_SIMD_66, vector_len);
  4295   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4255   } else {
  4296   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4256     emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_66, vector_len);
  4297   emit_int8(0x5E);
  4257   }
  4298   emit_operand(dst, src);
  4258 }
  4299 }
  4259 
  4300 
  4260 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4301 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4261   _instruction_uses_vl = true;
       
  4262   assert(VM_Version::supports_avx(), "");
  4302   assert(VM_Version::supports_avx(), "");
  4263   if (VM_Version::supports_evex()) {
  4303   InstructionMark im(this);
  4264     _tuple_type = EVEX_FV;
  4304   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4265     _input_size_in_bits = EVEX_32bit;
  4305   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
  4266   }
  4306   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4267   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_NONE, vector_len);
  4307   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  4308   emit_int8(0x5E);
       
  4309   emit_operand(dst, src);
  4268 }
  4310 }
  4269 
  4311 
  4270 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
  4312 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
  4271   _instruction_uses_vl = true;
       
  4272   assert(VM_Version::supports_avx(), "");
  4313   assert(VM_Version::supports_avx(), "");
  4273   if (VM_Version::supports_evex()) {
  4314   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4274     emit_vex_arith_q(0x51, dst, xnoreg, src, VEX_SIMD_66, vector_len);
  4315   int nds_enc = 0;
  4275   } else {
  4316   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4276     emit_vex_arith(0x51, dst, xnoreg, src, VEX_SIMD_66, vector_len);
  4317   emit_int8(0x51);
  4277   }
  4318   emit_int8((unsigned char)(0xC0 | encode));
  4278 }
  4319 }
  4279 
  4320 
  4280 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
  4321 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
  4281   _instruction_uses_vl = true;
       
  4282   assert(VM_Version::supports_avx(), "");
  4322   assert(VM_Version::supports_avx(), "");
  4283   if (VM_Version::supports_evex()) {
  4323   InstructionMark im(this);
  4284     _tuple_type = EVEX_FV;
  4324   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4285     _input_size_in_bits = EVEX_64bit;
  4325   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
  4286     emit_vex_arith_q(0x51, dst, xnoreg, src, VEX_SIMD_66, vector_len);
  4326   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4287   } else {
  4327   emit_int8(0x51);
  4288     emit_vex_arith(0x51, dst, xnoreg, src, VEX_SIMD_66, vector_len);
  4328   emit_operand(dst, src);
  4289   }
       
  4290 }
  4329 }
  4291 
  4330 
  4292 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
  4331 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
  4293   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4332   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4294   if (VM_Version::supports_avx512dq()) {
  4333   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
  4295     emit_simd_arith_q(0x54, dst, src, VEX_SIMD_66);
  4334   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4296   } else {
  4335   emit_int8(0x54);
  4297     emit_simd_arith(0x54, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ true);
  4336   emit_int8((unsigned char)(0xC0 | encode));
  4298   }
       
  4299 }
  4337 }
  4300 
  4338 
  4301 void Assembler::andps(XMMRegister dst, XMMRegister src) {
  4339 void Assembler::andps(XMMRegister dst, XMMRegister src) {
  4302   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  4340   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  4303   emit_simd_arith(0x54, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
  4341   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4342   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  4343   emit_int8(0x54);
       
  4344   emit_int8((unsigned char)(0xC0 | encode));
  4304 }
  4345 }
  4305 
  4346 
  4306 void Assembler::andps(XMMRegister dst, Address src) {
  4347 void Assembler::andps(XMMRegister dst, Address src) {
  4307   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  4348   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  4308   if (VM_Version::supports_evex()) {
  4349   InstructionMark im(this);
  4309     _tuple_type = EVEX_FV;
  4350   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
  4310     _input_size_in_bits = EVEX_32bit;
  4351   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
  4311   }
  4352   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
  4312   emit_simd_arith(0x54, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
  4353   emit_int8(0x54);
       
  4354   emit_operand(dst, src);
  4313 }
  4355 }
  4314 
  4356 
  4315 void Assembler::andpd(XMMRegister dst, Address src) {
  4357 void Assembler::andpd(XMMRegister dst, Address src) {
  4316   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4358   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4317   if (VM_Version::supports_avx512dq()) {
  4359   InstructionMark im(this);
  4318     _tuple_type = EVEX_FV;
  4360   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
  4319     _input_size_in_bits = EVEX_64bit;
  4361   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
  4320     emit_simd_arith_q(0x54, dst, src, VEX_SIMD_66);
  4362   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4321   } else {
  4363   emit_int8(0x54);
  4322     emit_simd_arith(0x54, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ true);
  4364   emit_operand(dst, src);
  4323   }
       
  4324 }
  4365 }
  4325 
  4366 
  4326 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4367 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4327   assert(VM_Version::supports_avx(), "");
  4368   assert(VM_Version::supports_avx(), "");
  4328   if (VM_Version::supports_avx512dq()) {
  4369   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
  4329     emit_vex_arith_q(0x54, dst, nds, src, VEX_SIMD_66, vector_len);
  4370   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4330   } else {
  4371   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4331     emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ true);
  4372   emit_int8(0x54);
  4332   }
  4373   emit_int8((unsigned char)(0xC0 | encode));
  4333 }
  4374 }
  4334 
  4375 
  4335 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4376 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4336   assert(VM_Version::supports_avx(), "");
  4377   assert(VM_Version::supports_avx(), "");
  4337   emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_NONE, vector_len, /* no_mask_reg */ false,  /* legacy_mode */ _legacy_mode_dq);
  4378   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4379   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4380   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  4381   emit_int8(0x54);
       
  4382   emit_int8((unsigned char)(0xC0 | encode));
  4338 }
  4383 }
  4339 
  4384 
  4340 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4385 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4341   assert(VM_Version::supports_avx(), "");
  4386   assert(VM_Version::supports_avx(), "");
  4342   if (VM_Version::supports_avx512dq()) {
  4387   InstructionMark im(this);
  4343     _tuple_type = EVEX_FV;
  4388   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
  4344     _input_size_in_bits = EVEX_64bit;
  4389   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
  4345     emit_vex_arith_q(0x54, dst, nds, src, VEX_SIMD_66, vector_len);
  4390   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4346   } else {
  4391   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4347     emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ true);
  4392   emit_int8(0x54);
  4348   }
  4393   emit_operand(dst, src);
  4349 }
  4394 }
  4350 
  4395 
  4351 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4396 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4352   assert(VM_Version::supports_avx(), "");
  4397   assert(VM_Version::supports_avx(), "");
  4353   if (VM_Version::supports_evex()) {
  4398   InstructionMark im(this);
  4354     _tuple_type = EVEX_FV;
  4399   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
  4355     _input_size_in_bits = EVEX_32bit;
  4400   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
  4356   }
  4401   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4357   emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_NONE, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
  4402   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  4403   emit_int8(0x54);
       
  4404   emit_operand(dst, src);
  4358 }
  4405 }
  4359 
  4406 
  4360 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
  4407 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
  4361   _instruction_uses_vl = true;
  4408   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4362   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4409   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4363   if (VM_Version::supports_evex()) {
  4410   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4364     emit_simd_arith_q(0x15, dst, src, VEX_SIMD_66);
  4411   emit_int8(0x15);
  4365   } else {
  4412   emit_int8((unsigned char)(0xC0 | encode));
  4366     emit_simd_arith(0x15, dst, src, VEX_SIMD_66);
       
  4367   }
       
  4368 }
  4413 }
  4369 
  4414 
  4370 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
  4415 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
  4371   _instruction_uses_vl = true;
  4416   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4372   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4417   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4373   if (VM_Version::supports_evex()) {
  4418   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4374     emit_simd_arith_q(0x14, dst, src, VEX_SIMD_66);
  4419   emit_int8(0x14);
  4375   } else {
  4420   emit_int8((unsigned char)(0xC0 | encode));
  4376     emit_simd_arith(0x14, dst, src, VEX_SIMD_66);
       
  4377   }
       
  4378 }
  4421 }
  4379 
  4422 
  4380 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
  4423 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
  4381   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4424   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4382   if (VM_Version::supports_avx512dq()) {
  4425   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
  4383     emit_simd_arith_q(0x57, dst, src, VEX_SIMD_66);
  4426   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4384   } else {
  4427   emit_int8(0x57);
  4385     emit_simd_arith(0x57, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ true);
  4428   emit_int8((unsigned char)(0xC0 | encode));
  4386   }
       
  4387 }
  4429 }
  4388 
  4430 
  4389 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
  4431 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
  4390   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  4432   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  4391   emit_simd_arith(0x57, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
  4433   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4434   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  4435   emit_int8(0x57);
       
  4436   emit_int8((unsigned char)(0xC0 | encode));
  4392 }
  4437 }
  4393 
  4438 
  4394 void Assembler::xorpd(XMMRegister dst, Address src) {
  4439 void Assembler::xorpd(XMMRegister dst, Address src) {
  4395   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4440   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4396   if (VM_Version::supports_avx512dq()) {
  4441   InstructionMark im(this);
  4397     _tuple_type = EVEX_FV;
  4442   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
  4398     _input_size_in_bits = EVEX_64bit;
  4443   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
  4399     emit_simd_arith_q(0x57, dst, src, VEX_SIMD_66);
  4444   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4400   } else {
  4445   emit_int8(0x57);
  4401     emit_simd_arith(0x57, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ true);
  4446   emit_operand(dst, src);
  4402   }
       
  4403 }
  4447 }
  4404 
  4448 
  4405 void Assembler::xorps(XMMRegister dst, Address src) {
  4449 void Assembler::xorps(XMMRegister dst, Address src) {
  4406   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  4450   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  4407   if (VM_Version::supports_evex()) {
  4451   InstructionMark im(this);
  4408     _tuple_type = EVEX_FV;
  4452   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
  4409     _input_size_in_bits = EVEX_32bit;
  4453   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
  4410   }
  4454   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
  4411   emit_simd_arith(0x57, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
  4455   emit_int8(0x57);
       
  4456   emit_operand(dst, src);
  4412 }
  4457 }
  4413 
  4458 
  4414 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4459 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4415   assert(VM_Version::supports_avx(), "");
  4460   assert(VM_Version::supports_avx(), "");
  4416   if (VM_Version::supports_avx512dq()) {
  4461   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
  4417     emit_vex_arith_q(0x57, dst, nds, src, VEX_SIMD_66, vector_len);
  4462   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4418   } else {
  4463   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4419     emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ true);
  4464   emit_int8(0x57);
  4420   }
  4465   emit_int8((unsigned char)(0xC0 | encode));
  4421 }
  4466 }
  4422 
  4467 
  4423 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4468 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4424   assert(VM_Version::supports_avx(), "");
  4469   assert(VM_Version::supports_avx(), "");
  4425   emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_NONE, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
  4470   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4471   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4472   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  4473   emit_int8(0x57);
       
  4474   emit_int8((unsigned char)(0xC0 | encode));
  4426 }
  4475 }
  4427 
  4476 
  4428 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4477 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4429   assert(VM_Version::supports_avx(), "");
  4478   assert(VM_Version::supports_avx(), "");
  4430   if (VM_Version::supports_avx512dq()) {
  4479   InstructionMark im(this);
  4431     _tuple_type = EVEX_FV;
  4480   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
  4432     _input_size_in_bits = EVEX_64bit;
  4481   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
  4433     emit_vex_arith_q(0x57, dst, nds, src, VEX_SIMD_66, vector_len);
  4482   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4434   } else {
  4483   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4435     emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ true);
  4484   emit_int8(0x57);
  4436   }
  4485   emit_operand(dst, src);
  4437 }
  4486 }
  4438 
  4487 
  4439 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4488 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4440   assert(VM_Version::supports_avx(), "");
  4489   assert(VM_Version::supports_avx(), "");
  4441   if (VM_Version::supports_evex()) {
  4490   InstructionMark im(this);
  4442     _tuple_type = EVEX_FV;
  4491   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
  4443     _input_size_in_bits = EVEX_32bit;
  4492   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
  4444   }
  4493   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4445   emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_NONE, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
  4494   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
       
  4495   emit_int8(0x57);
       
  4496   emit_operand(dst, src);
  4446 }
  4497 }
  4447 
  4498 
  4448 // Integer vector arithmetic
  4499 // Integer vector arithmetic
  4449 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4500 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4450   assert(VM_Version::supports_avx() && (vector_len == 0) ||
  4501   assert(VM_Version::supports_avx() && (vector_len == 0) ||
  4451          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
  4502          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
  4452   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38, /* legacy_mode */ true);
  4503   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  4504   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4505   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  4453   emit_int8(0x01);
  4506   emit_int8(0x01);
  4454   emit_int8((unsigned char)(0xC0 | encode));
  4507   emit_int8((unsigned char)(0xC0 | encode));
  4455 }
  4508 }
  4456 
  4509 
  4457 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4510 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4458   assert(VM_Version::supports_avx() && (vector_len == 0) ||
  4511   assert(VM_Version::supports_avx() && (vector_len == 0) ||
  4459          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
  4512          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
  4460   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38, /* legacy_mode */ true);
  4513   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  4514   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4515   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  4461   emit_int8(0x02);
  4516   emit_int8(0x02);
  4462   emit_int8((unsigned char)(0xC0 | encode));
  4517   emit_int8((unsigned char)(0xC0 | encode));
  4463 }
  4518 }
  4464 
  4519 
  4465 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
  4520 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
  4466   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4521   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4467   emit_simd_arith(0xFC, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  4522   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4523   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4524   emit_int8((unsigned char)0xFC);
       
  4525   emit_int8((unsigned char)(0xC0 | encode));
  4468 }
  4526 }
  4469 
  4527 
  4470 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
  4528 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
  4471   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4529   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4472   emit_simd_arith(0xFD, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  4530   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4531   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4532   emit_int8((unsigned char)0xFD);
       
  4533   emit_int8((unsigned char)(0xC0 | encode));
  4473 }
  4534 }
  4474 
  4535 
  4475 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
  4536 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
  4476   _instruction_uses_vl = true;
  4537   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4477   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4538   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4478   emit_simd_arith(0xFE, dst, src, VEX_SIMD_66);
  4539   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4540   emit_int8((unsigned char)0xFE);
       
  4541   emit_int8((unsigned char)(0xC0 | encode));
  4479 }
  4542 }
  4480 
  4543 
  4481 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
  4544 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
  4482   _instruction_uses_vl = true;
  4545   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4483   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4546   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4484   if (VM_Version::supports_evex()) {
  4547   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4485     emit_simd_arith_q(0xD4, dst, src, VEX_SIMD_66);
  4548   emit_int8((unsigned char)0xD4);
  4486   } else {
  4549   emit_int8((unsigned char)(0xC0 | encode));
  4487     emit_simd_arith(0xD4, dst, src, VEX_SIMD_66);
       
  4488   }
       
  4489 }
  4550 }
  4490 
  4551 
  4491 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
  4552 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
  4492   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
  4553   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
  4493   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
  4554   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  4494                                       VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
  4555   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  4495   emit_int8(0x01);
  4556   emit_int8(0x01);
  4496   emit_int8((unsigned char)(0xC0 | encode));
  4557   emit_int8((unsigned char)(0xC0 | encode));
  4497 }
  4558 }
  4498 
  4559 
  4499 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
  4560 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
  4500   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
  4561   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
  4501   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
  4562   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  4502                                       VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
  4563   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  4503   emit_int8(0x02);
  4564   emit_int8(0x02);
  4504   emit_int8((unsigned char)(0xC0 | encode));
  4565   emit_int8((unsigned char)(0xC0 | encode));
  4505 }
  4566 }
  4506 
  4567 
  4507 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4568 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4508   assert(UseAVX > 0, "requires some form of AVX");
  4569   assert(UseAVX > 0, "requires some form of AVX");
  4509   emit_vex_arith(0xFC, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  4570   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4571   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4572   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4573   emit_int8((unsigned char)0xFC);
       
  4574   emit_int8((unsigned char)(0xC0 | encode));
  4510 }
  4575 }
  4511 
  4576 
  4512 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4577 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4513   assert(UseAVX > 0, "requires some form of AVX");
  4578   assert(UseAVX > 0, "requires some form of AVX");
  4514   emit_vex_arith(0xFD, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  4579   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4580   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4581   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4582   emit_int8((unsigned char)0xFD);
       
  4583   emit_int8((unsigned char)(0xC0 | encode));
  4515 }
  4584 }
  4516 
  4585 
  4517 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4586 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4518   _instruction_uses_vl = true;
       
  4519   assert(UseAVX > 0, "requires some form of AVX");
  4587   assert(UseAVX > 0, "requires some form of AVX");
  4520   emit_vex_arith(0xFE, dst, nds, src, VEX_SIMD_66, vector_len);
  4588   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4589   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4590   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4591   emit_int8((unsigned char)0xFE);
       
  4592   emit_int8((unsigned char)(0xC0 | encode));
  4521 }
  4593 }
  4522 
  4594 
  4523 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4595 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4524   _instruction_uses_vl = true;
       
  4525   assert(UseAVX > 0, "requires some form of AVX");
  4596   assert(UseAVX > 0, "requires some form of AVX");
  4526   if (VM_Version::supports_evex()) {
  4597   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4527     emit_vex_arith_q(0xD4, dst, nds, src, VEX_SIMD_66, vector_len);
  4598   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4528   } else {
  4599   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4529     emit_vex_arith(0xD4, dst, nds, src, VEX_SIMD_66, vector_len);
  4600   emit_int8((unsigned char)0xD4);
  4530   }
  4601   emit_int8((unsigned char)(0xC0 | encode));
  4531 }
  4602 }
  4532 
  4603 
  4533 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4604 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4534   assert(UseAVX > 0, "requires some form of AVX");
  4605   assert(UseAVX > 0, "requires some form of AVX");
  4535   if (VM_Version::supports_evex()) {
  4606   InstructionMark im(this);
  4536     _tuple_type = EVEX_FVM;
  4607   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
  4537   }
  4608   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
  4538   emit_vex_arith(0xFC, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  4609   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4610   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4611   emit_int8((unsigned char)0xFC);
       
  4612   emit_operand(dst, src);
  4539 }
  4613 }
  4540 
  4614 
  4541 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4615 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4542   assert(UseAVX > 0, "requires some form of AVX");
  4616   assert(UseAVX > 0, "requires some form of AVX");
  4543   if (VM_Version::supports_evex()) {
  4617   InstructionMark im(this);
  4544     _tuple_type = EVEX_FVM;
  4618   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
  4545   }
  4619   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
  4546   emit_vex_arith(0xFD, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  4620   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4621   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4622   emit_int8((unsigned char)0xFD);
       
  4623   emit_operand(dst, src);
  4547 }
  4624 }
  4548 
  4625 
  4549 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4626 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4550   _instruction_uses_vl = true;
       
  4551   assert(UseAVX > 0, "requires some form of AVX");
  4627   assert(UseAVX > 0, "requires some form of AVX");
  4552   if (VM_Version::supports_evex()) {
  4628   InstructionMark im(this);
  4553     _tuple_type = EVEX_FV;
  4629   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4554     _input_size_in_bits = EVEX_32bit;
  4630   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
  4555   }
  4631   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4556   emit_vex_arith(0xFE, dst, nds, src, VEX_SIMD_66, vector_len);
  4632   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4633   emit_int8((unsigned char)0xFE);
       
  4634   emit_operand(dst, src);
  4557 }
  4635 }
  4558 
  4636 
  4559 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4637 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4560   _instruction_uses_vl = true;
       
  4561   assert(UseAVX > 0, "requires some form of AVX");
  4638   assert(UseAVX > 0, "requires some form of AVX");
  4562   if (VM_Version::supports_evex()) {
  4639   InstructionMark im(this);
  4563     _tuple_type = EVEX_FV;
  4640   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4564     _input_size_in_bits = EVEX_64bit;
  4641   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
  4565     emit_vex_arith_q(0xD4, dst, nds, src, VEX_SIMD_66, vector_len);
  4642   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4566   } else {
  4643   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4567     emit_vex_arith(0xD4, dst, nds, src, VEX_SIMD_66, vector_len);
  4644   emit_int8((unsigned char)0xD4);
  4568   }
  4645   emit_operand(dst, src);
  4569 }
  4646 }
  4570 
  4647 
  4571 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
  4648 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
  4572   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4649   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4573   emit_simd_arith(0xF8, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  4650   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4651   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4652   emit_int8((unsigned char)0xF8);
       
  4653   emit_int8((unsigned char)(0xC0 | encode));
  4574 }
  4654 }
  4575 
  4655 
  4576 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
  4656 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
  4577   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4657   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4578   emit_simd_arith(0xF9, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  4658   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4659   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4660   emit_int8((unsigned char)0xF9);
       
  4661   emit_int8((unsigned char)(0xC0 | encode));
  4579 }
  4662 }
  4580 
  4663 
  4581 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
  4664 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
  4582   _instruction_uses_vl = true;
  4665   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4583   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4666   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4584   emit_simd_arith(0xFA, dst, src, VEX_SIMD_66);
  4667   emit_int8((unsigned char)0xFA);
       
  4668   emit_int8((unsigned char)(0xC0 | encode));
  4585 }
  4669 }
  4586 
  4670 
  4587 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
  4671 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
  4588   _instruction_uses_vl = true;
  4672   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4589   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4673   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4590   if (VM_Version::supports_evex()) {
  4674   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4591     emit_simd_arith_q(0xFB, dst, src, VEX_SIMD_66);
  4675   emit_int8((unsigned char)0xFB);
  4592   } else {
  4676   emit_int8((unsigned char)(0xC0 | encode));
  4593     emit_simd_arith(0xFB, dst, src, VEX_SIMD_66);
       
  4594   }
       
  4595 }
  4677 }
  4596 
  4678 
  4597 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4679 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4598   assert(UseAVX > 0, "requires some form of AVX");
  4680   assert(UseAVX > 0, "requires some form of AVX");
  4599   emit_vex_arith(0xF8, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  4681   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4682   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4683   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4684   emit_int8((unsigned char)0xF8);
       
  4685   emit_int8((unsigned char)(0xC0 | encode));
  4600 }
  4686 }
  4601 
  4687 
  4602 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4688 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4603   assert(UseAVX > 0, "requires some form of AVX");
  4689   assert(UseAVX > 0, "requires some form of AVX");
  4604   emit_vex_arith(0xF9, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  4690   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4691   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4692   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4693   emit_int8((unsigned char)0xF9);
       
  4694   emit_int8((unsigned char)(0xC0 | encode));
  4605 }
  4695 }
  4606 
  4696 
  4607 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4697 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4608   _instruction_uses_vl = true;
       
  4609   assert(UseAVX > 0, "requires some form of AVX");
  4698   assert(UseAVX > 0, "requires some form of AVX");
  4610   emit_vex_arith(0xFA, dst, nds, src, VEX_SIMD_66, vector_len);
  4699   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4700   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4701   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4702   emit_int8((unsigned char)0xFA);
       
  4703   emit_int8((unsigned char)(0xC0 | encode));
  4611 }
  4704 }
  4612 
  4705 
  4613 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4706 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4614   _instruction_uses_vl = true;
       
  4615   assert(UseAVX > 0, "requires some form of AVX");
  4707   assert(UseAVX > 0, "requires some form of AVX");
  4616   if (VM_Version::supports_evex()) {
  4708   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4617     emit_vex_arith_q(0xFB, dst, nds, src, VEX_SIMD_66, vector_len);
  4709   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4618   } else {
  4710   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4619     emit_vex_arith(0xFB, dst, nds, src, VEX_SIMD_66, vector_len);
  4711   emit_int8((unsigned char)0xFB);
  4620   }
  4712   emit_int8((unsigned char)(0xC0 | encode));
  4621 }
  4713 }
  4622 
  4714 
  4623 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4715 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4624   assert(UseAVX > 0, "requires some form of AVX");
  4716   assert(UseAVX > 0, "requires some form of AVX");
  4625   if (VM_Version::supports_evex()) {
  4717   InstructionMark im(this);
  4626     _tuple_type = EVEX_FVM;
  4718   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
  4627   }
  4719   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
  4628   emit_vex_arith(0xF8, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  4720   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4721   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4722   emit_int8((unsigned char)0xF8);
       
  4723   emit_operand(dst, src);
  4629 }
  4724 }
  4630 
  4725 
  4631 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4726 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4632   assert(UseAVX > 0, "requires some form of AVX");
  4727   assert(UseAVX > 0, "requires some form of AVX");
  4633   if (VM_Version::supports_evex()) {
  4728   InstructionMark im(this);
  4634     _tuple_type = EVEX_FVM;
  4729   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
  4635   }
  4730   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
  4636   emit_vex_arith(0xF9, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  4731   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4732   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4733   emit_int8((unsigned char)0xF9);
       
  4734   emit_operand(dst, src);
  4637 }
  4735 }
  4638 
  4736 
  4639 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4737 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4640   _instruction_uses_vl = true;
       
  4641   assert(UseAVX > 0, "requires some form of AVX");
  4738   assert(UseAVX > 0, "requires some form of AVX");
  4642   if (VM_Version::supports_evex()) {
  4739   InstructionMark im(this);
  4643     _tuple_type = EVEX_FV;
  4740   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4644     _input_size_in_bits = EVEX_32bit;
  4741   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
  4645   }
  4742   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4646   emit_vex_arith(0xFA, dst, nds, src, VEX_SIMD_66, vector_len);
  4743   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4744   emit_int8((unsigned char)0xFA);
       
  4745   emit_operand(dst, src);
  4647 }
  4746 }
  4648 
  4747 
  4649 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4748 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4650   _instruction_uses_vl = true;
       
  4651   assert(UseAVX > 0, "requires some form of AVX");
  4749   assert(UseAVX > 0, "requires some form of AVX");
  4652   if (VM_Version::supports_evex()) {
  4750   InstructionMark im(this);
  4653     _tuple_type = EVEX_FV;
  4751   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4654     _input_size_in_bits = EVEX_64bit;
  4752   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
  4655     emit_vex_arith_q(0xFB, dst, nds, src, VEX_SIMD_66, vector_len);
  4753   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4656   } else {
  4754   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4657     emit_vex_arith(0xFB, dst, nds, src, VEX_SIMD_66, vector_len);
  4755   emit_int8((unsigned char)0xFB);
  4658   }
  4756   emit_operand(dst, src);
  4659 }
  4757 }
  4660 
  4758 
  4661 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
  4759 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
  4662   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4760   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4663   emit_simd_arith(0xD5, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  4761   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4762   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4763   emit_int8((unsigned char)0xD5);
       
  4764   emit_int8((unsigned char)(0xC0 | encode));
  4664 }
  4765 }
  4665 
  4766 
  4666 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
  4767 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
  4667   _instruction_uses_vl = true;
       
  4668   assert(VM_Version::supports_sse4_1(), "");
  4768   assert(VM_Version::supports_sse4_1(), "");
  4669   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66,
  4769   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4670                                       /* no_mask_reg */ false, VEX_OPCODE_0F_38);
  4770   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  4671   emit_int8(0x40);
  4771   emit_int8(0x40);
  4672   emit_int8((unsigned char)(0xC0 | encode));
  4772   emit_int8((unsigned char)(0xC0 | encode));
  4673 }
  4773 }
  4674 
  4774 
  4675 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4775 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4676   assert(UseAVX > 0, "requires some form of AVX");
  4776   assert(UseAVX > 0, "requires some form of AVX");
  4677   emit_vex_arith(0xD5, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  4777   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4778   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4779   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4780   emit_int8((unsigned char)0xD5);
       
  4781   emit_int8((unsigned char)(0xC0 | encode));
  4678 }
  4782 }
  4679 
  4783 
  4680 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4784 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4681   _instruction_uses_vl = true;
       
  4682   assert(UseAVX > 0, "requires some form of AVX");
  4785   assert(UseAVX > 0, "requires some form of AVX");
  4683   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);
  4786   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4787   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4788   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  4684   emit_int8(0x40);
  4789   emit_int8(0x40);
  4685   emit_int8((unsigned char)(0xC0 | encode));
  4790   emit_int8((unsigned char)(0xC0 | encode));
  4686 }
  4791 }
  4687 
  4792 
  4688 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4793 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  4689   assert(UseAVX > 2, "requires some form of AVX");
  4794   assert(UseAVX > 2, "requires some form of AVX");
  4690   int src_enc = src->encoding();
  4795   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
  4691   int dst_enc = dst->encoding();
  4796   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4692   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4797   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  4693   int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_38,
       
  4694                                      /* vex_w */ true, vector_len, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false);
       
  4695   emit_int8(0x40);
  4798   emit_int8(0x40);
  4696   emit_int8((unsigned char)(0xC0 | encode));
  4799   emit_int8((unsigned char)(0xC0 | encode));
  4697 }
  4800 }
  4698 
  4801 
  4699 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4802 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4700   assert(UseAVX > 0, "requires some form of AVX");
  4803   assert(UseAVX > 0, "requires some form of AVX");
  4701   if (VM_Version::supports_evex()) {
  4804   InstructionMark im(this);
  4702     _tuple_type = EVEX_FVM;
  4805   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
  4703   }
  4806   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
  4704   emit_vex_arith(0xD5, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  4807   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4808   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4809   emit_int8((unsigned char)0xD5);
       
  4810   emit_operand(dst, src);
  4705 }
  4811 }
  4706 
  4812 
  4707 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4813 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4708   _instruction_uses_vl = true;
       
  4709   assert(UseAVX > 0, "requires some form of AVX");
  4814   assert(UseAVX > 0, "requires some form of AVX");
  4710   if (VM_Version::supports_evex()) {
  4815   InstructionMark im(this);
  4711     _tuple_type = EVEX_FV;
  4816   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4712     _input_size_in_bits = EVEX_32bit;
  4817   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
  4713   }
  4818   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4714   InstructionMark im(this);
  4819   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  4715   int dst_enc = dst->encoding();
       
  4716   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4717   vex_prefix(src, nds_enc, dst_enc, VEX_SIMD_66,
       
  4718              VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
       
  4719   emit_int8(0x40);
  4820   emit_int8(0x40);
  4720   emit_operand(dst, src);
  4821   emit_operand(dst, src);
  4721 }
  4822 }
  4722 
  4823 
  4723 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4824 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  4724   assert(UseAVX > 0, "requires some form of AVX");
  4825   assert(UseAVX > 0, "requires some form of AVX");
  4725   if (VM_Version::supports_evex()) {
  4826   InstructionMark im(this);
  4726     _tuple_type = EVEX_FV;
  4827   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
  4727     _input_size_in_bits = EVEX_64bit;
  4828   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
  4728   }
  4829   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  4729   InstructionMark im(this);
  4830   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  4730   int dst_enc = dst->encoding();
       
  4731   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  4732   vex_prefix(src, nds_enc, dst_enc, VEX_SIMD_66,
       
  4733              VEX_OPCODE_0F_38, /* vex_w */ true, vector_len, /* legacy_mode */ _legacy_mode_dq);
       
  4734   emit_int8(0x40);
  4831   emit_int8(0x40);
  4735   emit_operand(dst, src);
  4832   emit_operand(dst, src);
  4736 }
  4833 }
  4737 
  4834 
  4738 // Shift packed integers left by specified number of bits.
  4835 // Shift packed integers left by specified number of bits.
  4739 void Assembler::psllw(XMMRegister dst, int shift) {
  4836 void Assembler::psllw(XMMRegister dst, int shift) {
  4740   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4837   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
       
  4838   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
  4741   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
  4839   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
  4742   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F,
  4840   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4743                                       /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
       
  4744   emit_int8(0x71);
  4841   emit_int8(0x71);
  4745   emit_int8((unsigned char)(0xC0 | encode));
  4842   emit_int8((unsigned char)(0xC0 | encode));
  4746   emit_int8(shift & 0xFF);
  4843   emit_int8(shift & 0xFF);
  4747 }
  4844 }
  4748 
  4845 
  4749 void Assembler::pslld(XMMRegister dst, int shift) {
  4846 void Assembler::pslld(XMMRegister dst, int shift) {
  4750   _instruction_uses_vl = true;
  4847   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4751   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4848   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4752   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
  4849   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
  4753   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false);
  4850   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4754   emit_int8(0x72);
  4851   emit_int8(0x72);
  4755   emit_int8((unsigned char)(0xC0 | encode));
  4852   emit_int8((unsigned char)(0xC0 | encode));
  4756   emit_int8(shift & 0xFF);
  4853   emit_int8(shift & 0xFF);
  4757 }
  4854 }
  4758 
  4855 
  4759 void Assembler::psllq(XMMRegister dst, int shift) {
  4856 void Assembler::psllq(XMMRegister dst, int shift) {
  4760   _instruction_uses_vl = true;
  4857   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4761   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4858   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4762   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
  4859   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
  4763   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F, /* rex_w */ true);
  4860   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4764   emit_int8(0x73);
  4861   emit_int8(0x73);
  4765   emit_int8((unsigned char)(0xC0 | encode));
  4862   emit_int8((unsigned char)(0xC0 | encode));
  4766   emit_int8(shift & 0xFF);
  4863   emit_int8(shift & 0xFF);
  4767 }
  4864 }
  4768 
  4865 
  4769 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
  4866 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
  4770   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4867   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4771   emit_simd_arith(0xF1, dst, shift, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  4868   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4869   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4870   emit_int8((unsigned char)0xF1);
       
  4871   emit_int8((unsigned char)(0xC0 | encode));
  4772 }
  4872 }
  4773 
  4873 
  4774 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
  4874 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
  4775   _instruction_uses_vl = true;
  4875   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4776   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4876   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4777   emit_simd_arith(0xF2, dst, shift, VEX_SIMD_66);
  4877   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4878   emit_int8((unsigned char)0xF2);
       
  4879   emit_int8((unsigned char)(0xC0 | encode));
  4778 }
  4880 }
  4779 
  4881 
  4780 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
  4882 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
  4781   _instruction_uses_vl = true;
  4883   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4782   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4884   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4783   if (VM_Version::supports_evex()) {
  4885   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4784     emit_simd_arith_q(0xF3, dst, shift, VEX_SIMD_66);
  4886   emit_int8((unsigned char)0xF3);
  4785   } else {
  4887   emit_int8((unsigned char)(0xC0 | encode));
  4786     emit_simd_arith(0xF3, dst, shift, VEX_SIMD_66);
       
  4787   }
       
  4788 }
  4888 }
  4789 
  4889 
  4790 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
  4890 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
  4791   assert(UseAVX > 0, "requires some form of AVX");
  4891   assert(UseAVX > 0, "requires some form of AVX");
       
  4892   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
  4792   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
  4893   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
  4793   emit_vex_arith(0x71, xmm6, dst, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  4894   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4895   emit_int8(0x71);
       
  4896   emit_int8((unsigned char)(0xC0 | encode));
  4794   emit_int8(shift & 0xFF);
  4897   emit_int8(shift & 0xFF);
  4795 }
  4898 }
  4796 
  4899 
  4797 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
  4900 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
  4798   _instruction_uses_vl = true;
       
  4799   assert(UseAVX > 0, "requires some form of AVX");
  4901   assert(UseAVX > 0, "requires some form of AVX");
       
  4902   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
       
  4903   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4800   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
  4904   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
  4801   emit_vex_arith(0x72, xmm6, dst, src, VEX_SIMD_66, vector_len);
  4905   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4906   emit_int8(0x72);
       
  4907   emit_int8((unsigned char)(0xC0 | encode));
  4802   emit_int8(shift & 0xFF);
  4908   emit_int8(shift & 0xFF);
  4803 }
  4909 }
  4804 
  4910 
  4805 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
  4911 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
  4806   _instruction_uses_vl = true;
       
  4807   assert(UseAVX > 0, "requires some form of AVX");
  4912   assert(UseAVX > 0, "requires some form of AVX");
       
  4913   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4808   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
  4914   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
  4809   if (VM_Version::supports_evex()) {
  4915   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4810     emit_vex_arith_q(0x73, xmm6, dst, src, VEX_SIMD_66, vector_len);
  4916   emit_int8(0x73);
  4811   } else {
  4917   emit_int8((unsigned char)(0xC0 | encode));
  4812     emit_vex_arith(0x73, xmm6, dst, src, VEX_SIMD_66, vector_len);
       
  4813   }
       
  4814   emit_int8(shift & 0xFF);
  4918   emit_int8(shift & 0xFF);
  4815 }
  4919 }
  4816 
  4920 
  4817 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
  4921 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
  4818   assert(UseAVX > 0, "requires some form of AVX");
  4922   assert(UseAVX > 0, "requires some form of AVX");
  4819   emit_vex_arith(0xF1, dst, src, shift, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  4923   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4924   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4925   emit_int8((unsigned char)0xF1);
       
  4926   emit_int8((unsigned char)(0xC0 | encode));
  4820 }
  4927 }
  4821 
  4928 
  4822 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
  4929 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
  4823   _instruction_uses_vl = true;
       
  4824   assert(UseAVX > 0, "requires some form of AVX");
  4930   assert(UseAVX > 0, "requires some form of AVX");
  4825   emit_vex_arith(0xF2, dst, src, shift, VEX_SIMD_66, vector_len);
  4931   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4932   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4933   emit_int8((unsigned char)0xF2);
       
  4934   emit_int8((unsigned char)(0xC0 | encode));
  4826 }
  4935 }
  4827 
  4936 
  4828 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
  4937 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
  4829   _instruction_uses_vl = true;
       
  4830   assert(UseAVX > 0, "requires some form of AVX");
  4938   assert(UseAVX > 0, "requires some form of AVX");
  4831   if (VM_Version::supports_evex()) {
  4939   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4832     emit_vex_arith_q(0xF3, dst, src, shift, VEX_SIMD_66, vector_len);
  4940   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4833   } else {
  4941   emit_int8((unsigned char)0xF3);
  4834     emit_vex_arith(0xF3, dst, src, shift, VEX_SIMD_66, vector_len);
  4942   emit_int8((unsigned char)(0xC0 | encode));
  4835   }
       
  4836 }
  4943 }
  4837 
  4944 
  4838 // Shift packed integers logically right by specified number of bits.
  4945 // Shift packed integers logically right by specified number of bits.
  4839 void Assembler::psrlw(XMMRegister dst, int shift) {
  4946 void Assembler::psrlw(XMMRegister dst, int shift) {
  4840   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4947   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
       
  4948   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
  4841   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
  4949   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
  4842   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false,
  4950   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4843                                       VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
       
  4844   emit_int8(0x71);
  4951   emit_int8(0x71);
  4845   emit_int8((unsigned char)(0xC0 | encode));
  4952   emit_int8((unsigned char)(0xC0 | encode));
  4846   emit_int8(shift & 0xFF);
  4953   emit_int8(shift & 0xFF);
  4847 }
  4954 }
  4848 
  4955 
  4849 void Assembler::psrld(XMMRegister dst, int shift) {
  4956 void Assembler::psrld(XMMRegister dst, int shift) {
  4850   _instruction_uses_vl = true;
  4957   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4851   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4958   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4852   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
  4959   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
  4853   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false);
  4960   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4854   emit_int8(0x72);
  4961   emit_int8(0x72);
  4855   emit_int8((unsigned char)(0xC0 | encode));
  4962   emit_int8((unsigned char)(0xC0 | encode));
  4856   emit_int8(shift & 0xFF);
  4963   emit_int8(shift & 0xFF);
  4857 }
  4964 }
  4858 
  4965 
  4859 void Assembler::psrlq(XMMRegister dst, int shift) {
  4966 void Assembler::psrlq(XMMRegister dst, int shift) {
  4860   _instruction_uses_vl = true;
       
  4861   // Do not confuse it with psrldq SSE2 instruction which
  4967   // Do not confuse it with psrldq SSE2 instruction which
  4862   // shifts 128 bit value in xmm register by number of bytes.
  4968   // shifts 128 bit value in xmm register by number of bytes.
  4863   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4969   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
       
  4970   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4864   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
  4971   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
  4865   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false,
  4972   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4866                                       VEX_OPCODE_0F, /* rex_w */ VM_Version::supports_evex());
       
  4867   emit_int8(0x73);
  4973   emit_int8(0x73);
  4868   emit_int8((unsigned char)(0xC0 | encode));
  4974   emit_int8((unsigned char)(0xC0 | encode));
  4869   emit_int8(shift & 0xFF);
  4975   emit_int8(shift & 0xFF);
  4870 }
  4976 }
  4871 
  4977 
  4872 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
  4978 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
  4873   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4979   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4874   emit_simd_arith(0xD1, dst, shift, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  4980   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
       
  4981   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4982   emit_int8((unsigned char)0xD1);
       
  4983   emit_int8((unsigned char)(0xC0 | encode));
  4875 }
  4984 }
  4876 
  4985 
  4877 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
  4986 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
  4878   _instruction_uses_vl = true;
  4987   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4879   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4988   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4880   emit_simd_arith(0xD2, dst, shift, VEX_SIMD_66);
  4989   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  4990   emit_int8((unsigned char)0xD2);
       
  4991   emit_int8((unsigned char)(0xC0 | encode));
  4881 }
  4992 }
  4882 
  4993 
  4883 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
  4994 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
  4884   _instruction_uses_vl = true;
  4995   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4885   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4996   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4886   if (VM_Version::supports_evex()) {
  4997   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4887     emit_simd_arith_q(0xD3, dst, shift, VEX_SIMD_66);
  4998   emit_int8((unsigned char)0xD3);
  4888   } else {
  4999   emit_int8((unsigned char)(0xC0 | encode));
  4889     emit_simd_arith(0xD3, dst, shift, VEX_SIMD_66);
       
  4890   }
       
  4891 }
  5000 }
  4892 
  5001 
  4893 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
  5002 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
  4894   assert(UseAVX > 0, "requires some form of AVX");
  5003   assert(UseAVX > 0, "requires some form of AVX");
       
  5004   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
  4895   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
  5005   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
  4896   emit_vex_arith(0x71, xmm2, dst, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  5006   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  5007   emit_int8(0x71);
       
  5008   emit_int8((unsigned char)(0xC0 | encode));
  4897   emit_int8(shift & 0xFF);
  5009   emit_int8(shift & 0xFF);
  4898 }
  5010 }
  4899 
  5011 
  4900 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
  5012 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
  4901   _instruction_uses_vl = true;
       
  4902   assert(UseAVX > 0, "requires some form of AVX");
  5013   assert(UseAVX > 0, "requires some form of AVX");
       
  5014   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4903   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
  5015   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
  4904   emit_vex_arith(0x72, xmm2, dst, src, VEX_SIMD_66, vector_len);
  5016   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  5017   emit_int8(0x72);
       
  5018   emit_int8((unsigned char)(0xC0 | encode));
  4905   emit_int8(shift & 0xFF);
  5019   emit_int8(shift & 0xFF);
  4906 }
  5020 }
  4907 
  5021 
  4908 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
  5022 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
  4909   _instruction_uses_vl = true;
       
  4910   assert(UseAVX > 0, "requires some form of AVX");
  5023   assert(UseAVX > 0, "requires some form of AVX");
       
  5024   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4911   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
  5025   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
  4912   if (VM_Version::supports_evex()) {
  5026   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4913     emit_vex_arith_q(0x73, xmm2, dst, src, VEX_SIMD_66, vector_len);
  5027   emit_int8(0x73);
  4914   } else {
  5028   emit_int8((unsigned char)(0xC0 | encode));
  4915     emit_vex_arith(0x73, xmm2, dst, src, VEX_SIMD_66, vector_len);
       
  4916   }
       
  4917   emit_int8(shift & 0xFF);
  5029   emit_int8(shift & 0xFF);
  4918 }
  5030 }
  4919 
  5031 
  4920 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
  5032 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
  4921   assert(UseAVX > 0, "requires some form of AVX");
  5033   assert(UseAVX > 0, "requires some form of AVX");
  4922   emit_vex_arith(0xD1, dst, src, shift, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  5034   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
       
  5035   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  5036   emit_int8((unsigned char)0xD1);
       
  5037   emit_int8((unsigned char)(0xC0 | encode));
  4923 }
  5038 }
  4924 
  5039 
  4925 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
  5040 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
  4926   _instruction_uses_vl = true;
       
  4927   assert(UseAVX > 0, "requires some form of AVX");
  5041   assert(UseAVX > 0, "requires some form of AVX");
  4928   emit_vex_arith(0xD2, dst, src, shift, VEX_SIMD_66, vector_len);
  5042   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  5043   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  5044   emit_int8((unsigned char)0xD2);
       
  5045   emit_int8((unsigned char)(0xC0 | encode));
  4929 }
  5046 }
  4930 
  5047 
  4931 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
  5048 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
  4932   _instruction_uses_vl = true;
       
  4933   assert(UseAVX > 0, "requires some form of AVX");
  5049   assert(UseAVX > 0, "requires some form of AVX");
  4934   if (VM_Version::supports_evex()) {
  5050   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4935     emit_vex_arith_q(0xD3, dst, src, shift, VEX_SIMD_66, vector_len);
  5051   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4936   } else {
  5052   emit_int8((unsigned char)0xD3);
  4937     emit_vex_arith(0xD3, dst, src, shift, VEX_SIMD_66, vector_len);
  5053   emit_int8((unsigned char)(0xC0 | encode));
  4938   }
       
  4939 }
  5054 }
  4940 
  5055 
  4941 // Shift packed integers arithmetically right by specified number of bits.
  5056 // Shift packed integers arithmetically right by specified number of bits.
  4942 void Assembler::psraw(XMMRegister dst, int shift) {
  5057 void Assembler::psraw(XMMRegister dst, int shift) {
  4943   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  5058   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
       
  5059   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
  4944   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
  5060   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
  4945   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false,
  5061   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4946                                       VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
       
  4947   emit_int8(0x71);
  5062   emit_int8(0x71);
  4948   emit_int8((unsigned char)(0xC0 | encode));
  5063   emit_int8((unsigned char)(0xC0 | encode));
  4949   emit_int8(shift & 0xFF);
  5064   emit_int8(shift & 0xFF);
  4950 }
  5065 }
  4951 
  5066 
  4952 void Assembler::psrad(XMMRegister dst, int shift) {
  5067 void Assembler::psrad(XMMRegister dst, int shift) {
  4953   _instruction_uses_vl = true;
  5068   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4954   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  5069   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4955   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
  5070   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
  4956   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false);
  5071   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  4957   emit_int8(0x72);
  5072   emit_int8(0x72);
  4958   emit_int8((unsigned char)(0xC0 | encode));
  5073   emit_int8((unsigned char)(0xC0 | encode));
  4959   emit_int8(shift & 0xFF);
  5074   emit_int8(shift & 0xFF);
  4960 }
  5075 }
  4961 
  5076 
  4962 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
  5077 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
  4963   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  5078   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4964   emit_simd_arith(0xE1, dst, shift, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  5079   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
       
  5080   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  5081   emit_int8((unsigned char)0xE1);
       
  5082   emit_int8((unsigned char)(0xC0 | encode));
  4965 }
  5083 }
  4966 
  5084 
  4967 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
  5085 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
  4968   _instruction_uses_vl = true;
  5086   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  4969   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  5087   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4970   emit_simd_arith(0xE2, dst, shift, VEX_SIMD_66);
  5088   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  5089   emit_int8((unsigned char)0xE2);
       
  5090   emit_int8((unsigned char)(0xC0 | encode));
  4971 }
  5091 }
  4972 
  5092 
  4973 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
  5093 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
  4974   assert(UseAVX > 0, "requires some form of AVX");
  5094   assert(UseAVX > 0, "requires some form of AVX");
       
  5095   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
  4975   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
  5096   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
  4976   emit_vex_arith(0x71, xmm4, dst, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  5097   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  5098   emit_int8(0x71);
       
  5099   emit_int8((unsigned char)(0xC0 | encode));
  4977   emit_int8(shift & 0xFF);
  5100   emit_int8(shift & 0xFF);
  4978 }
  5101 }
  4979 
  5102 
  4980 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
  5103 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
  4981   _instruction_uses_vl = true;
       
  4982   assert(UseAVX > 0, "requires some form of AVX");
  5104   assert(UseAVX > 0, "requires some form of AVX");
       
  5105   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  4983   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
  5106   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
  4984   emit_vex_arith(0x72, xmm4, dst, src, VEX_SIMD_66, vector_len);
  5107   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  5108   emit_int8(0x72);
       
  5109   emit_int8((unsigned char)(0xC0 | encode));
  4985   emit_int8(shift & 0xFF);
  5110   emit_int8(shift & 0xFF);
  4986 }
  5111 }
  4987 
  5112 
  4988 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
  5113 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
  4989   assert(UseAVX > 0, "requires some form of AVX");
  5114   assert(UseAVX > 0, "requires some form of AVX");
  4990   emit_vex_arith(0xE1, dst, src, shift, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
  5115   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
       
  5116   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  5117   emit_int8((unsigned char)0xE1);
       
  5118   emit_int8((unsigned char)(0xC0 | encode));
  4991 }
  5119 }
  4992 
  5120 
  4993 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
  5121 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
  4994   _instruction_uses_vl = true;
       
  4995   assert(UseAVX > 0, "requires some form of AVX");
  5122   assert(UseAVX > 0, "requires some form of AVX");
  4996   emit_vex_arith(0xE2, dst, src, shift, VEX_SIMD_66, vector_len);
  5123   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  5124   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  5125   emit_int8((unsigned char)0xE2);
       
  5126   emit_int8((unsigned char)(0xC0 | encode));
  4997 }
  5127 }
  4998 
  5128 
  4999 
  5129 
  5000 // logical operations packed integers
  5130 // logical operations packed integers
  5001 void Assembler::pand(XMMRegister dst, XMMRegister src) {
  5131 void Assembler::pand(XMMRegister dst, XMMRegister src) {
  5002   _instruction_uses_vl = true;
  5132   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  5003   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  5133   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  5004   emit_simd_arith(0xDB, dst, src, VEX_SIMD_66);
  5134   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  5135   emit_int8((unsigned char)0xDB);
       
  5136   emit_int8((unsigned char)(0xC0 | encode));
  5005 }
  5137 }
  5006 
  5138 
  5007 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  5139 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  5008   _instruction_uses_vl = true;
       
  5009   assert(UseAVX > 0, "requires some form of AVX");
  5140   assert(UseAVX > 0, "requires some form of AVX");
  5010   emit_vex_arith(0xDB, dst, nds, src, VEX_SIMD_66, vector_len);
  5141   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  5142   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  5143   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  5144   emit_int8((unsigned char)0xDB);
       
  5145   emit_int8((unsigned char)(0xC0 | encode));
  5011 }
  5146 }
  5012 
  5147 
  5013 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  5148 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  5014   _instruction_uses_vl = true;
       
  5015   assert(UseAVX > 0, "requires some form of AVX");
  5149   assert(UseAVX > 0, "requires some form of AVX");
  5016   if (VM_Version::supports_evex()) {
  5150   InstructionMark im(this);
  5017     _tuple_type = EVEX_FV;
  5151   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  5018     _input_size_in_bits = EVEX_32bit;
  5152   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
  5019   }
  5153   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  5020   emit_vex_arith(0xDB, dst, nds, src, VEX_SIMD_66, vector_len);
  5154   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  5155   emit_int8((unsigned char)0xDB);
       
  5156   emit_operand(dst, src);
  5021 }
  5157 }
  5022 
  5158 
  5023 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
  5159 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
  5024   _instruction_uses_vl = true;
  5160   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  5025   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  5161   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  5026   if (VM_Version::supports_evex()) {
  5162   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  5027     emit_simd_arith_q(0xDF, dst, src, VEX_SIMD_66);
  5163   emit_int8((unsigned char)0xDF);
  5028   }
  5164   emit_int8((unsigned char)(0xC0 | encode));
  5029   else {
       
  5030     emit_simd_arith(0xDF, dst, src, VEX_SIMD_66);
       
  5031   }
       
  5032 }
  5165 }
  5033 
  5166 
  5034 void Assembler::por(XMMRegister dst, XMMRegister src) {
  5167 void Assembler::por(XMMRegister dst, XMMRegister src) {
  5035   _instruction_uses_vl = true;
  5168   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  5036   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  5169   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  5037   emit_simd_arith(0xEB, dst, src, VEX_SIMD_66);
  5170   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  5171   emit_int8((unsigned char)0xEB);
       
  5172   emit_int8((unsigned char)(0xC0 | encode));
  5038 }
  5173 }
  5039 
  5174 
  5040 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  5175 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  5041   _instruction_uses_vl = true;
       
  5042   assert(UseAVX > 0, "requires some form of AVX");
  5176   assert(UseAVX > 0, "requires some form of AVX");
  5043   emit_vex_arith(0xEB, dst, nds, src, VEX_SIMD_66, vector_len);
  5177   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  5178   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  5179   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  5180   emit_int8((unsigned char)0xEB);
       
  5181   emit_int8((unsigned char)(0xC0 | encode));
  5044 }
  5182 }
  5045 
  5183 
  5046 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  5184 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  5047   _instruction_uses_vl = true;
       
  5048   assert(UseAVX > 0, "requires some form of AVX");
  5185   assert(UseAVX > 0, "requires some form of AVX");
  5049   if (VM_Version::supports_evex()) {
  5186   InstructionMark im(this);
  5050     _tuple_type = EVEX_FV;
  5187   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  5051     _input_size_in_bits = EVEX_32bit;
  5188   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
  5052   }
  5189   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  5053   emit_vex_arith(0xEB, dst, nds, src, VEX_SIMD_66, vector_len);
  5190   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  5191   emit_int8((unsigned char)0xEB);
       
  5192   emit_operand(dst, src);
  5054 }
  5193 }
  5055 
  5194 
  5056 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
  5195 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
  5057   _instruction_uses_vl = true;
  5196   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  5058   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  5197   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  5059   emit_simd_arith(0xEF, dst, src, VEX_SIMD_66);
  5198   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  5199   emit_int8((unsigned char)0xEF);
       
  5200   emit_int8((unsigned char)(0xC0 | encode));
  5060 }
  5201 }
  5061 
  5202 
  5062 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  5203 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
  5063   _instruction_uses_vl = true;
       
  5064   assert(UseAVX > 0, "requires some form of AVX");
  5204   assert(UseAVX > 0, "requires some form of AVX");
  5065   emit_vex_arith(0xEF, dst, nds, src, VEX_SIMD_66, vector_len);
  5205   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  5206   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  5207   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  5208   emit_int8((unsigned char)0xEF);
       
  5209   emit_int8((unsigned char)(0xC0 | encode));
  5066 }
  5210 }
  5067 
  5211 
  5068 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  5212 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
  5069   _instruction_uses_vl = true;
       
  5070   assert(UseAVX > 0, "requires some form of AVX");
  5213   assert(UseAVX > 0, "requires some form of AVX");
  5071   if (VM_Version::supports_evex()) {
  5214   InstructionMark im(this);
  5072     _tuple_type = EVEX_FV;
  5215   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  5073     _input_size_in_bits = EVEX_32bit;
  5216   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
  5074   }
  5217   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  5075   emit_vex_arith(0xEF, dst, nds, src, VEX_SIMD_66, vector_len);
  5218   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
       
  5219   emit_int8((unsigned char)0xEF);
       
  5220   emit_operand(dst, src);
  5076 }
  5221 }
  5077 
  5222 
  5078 
  5223 
  5079 void Assembler::vinsertf128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  5224 void Assembler::vinsertf128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  5080   assert(VM_Version::supports_avx(), "");
  5225   assert(VM_Version::supports_avx(), "");
  5081   int vector_len = AVX_256bit;
  5226   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
  5082   if (VM_Version::supports_evex()) {
  5227   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  5083     vector_len = AVX_512bit;
  5228   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  5084   }
  5229   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5085   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A);
       
  5086   emit_int8(0x18);
  5230   emit_int8(0x18);
  5087   emit_int8((unsigned char)(0xC0 | encode));
  5231   emit_int8((unsigned char)(0xC0 | encode));
  5088   // 0x00 - insert into lower 128 bits
  5232   // 0x00 - insert into lower 128 bits
  5089   // 0x01 - insert into upper 128 bits
  5233   // 0x01 - insert into upper 128 bits
  5090   emit_int8(0x01);
  5234   emit_int8(0x01);
  5091 }
  5235 }
  5092 
  5236 
  5093 void Assembler::vinsertf64x4h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  5237 void Assembler::vinsertf64x4h(XMMRegister dst, XMMRegister nds, XMMRegister src, int value) {
  5094   assert(VM_Version::supports_evex(), "");
  5238   assert(VM_Version::supports_evex(), "");
  5095   int vector_len = AVX_512bit;
  5239   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  5096   int src_enc = src->encoding();
  5240   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  5097   int dst_enc = dst->encoding();
  5241   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5098   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  5099   int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
       
  5100                                      /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
       
  5101   emit_int8(0x1A);
  5242   emit_int8(0x1A);
  5102   emit_int8((unsigned char)(0xC0 | encode));
  5243   emit_int8((unsigned char)(0xC0 | encode));
  5103   // 0x00 - insert into lower 256 bits
  5244   // 0x00 - insert into lower 256 bits
  5104   // 0x01 - insert into upper 256 bits
  5245   // 0x01 - insert into upper 256 bits
  5105   emit_int8(0x01);
  5246   emit_int8(value & 0x01);
  5106 }
  5247 }
  5107 
  5248 
  5108 void Assembler::vinsertf64x4h(XMMRegister dst, Address src) {
  5249 void Assembler::vinsertf64x4h(XMMRegister dst, Address src, int value) {
  5109   assert(VM_Version::supports_evex(), "");
  5250   assert(VM_Version::supports_evex(), "");
  5110   _tuple_type = EVEX_T4;
       
  5111   _input_size_in_bits = EVEX_64bit;
       
  5112   InstructionMark im(this);
       
  5113   int vector_len = AVX_512bit;
       
  5114   assert(dst != xnoreg, "sanity");
  5251   assert(dst != xnoreg, "sanity");
  5115   int dst_enc = dst->encoding();
  5252   InstructionMark im(this);
       
  5253   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
       
  5254   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
  5116   // swap src<->dst for encoding
  5255   // swap src<->dst for encoding
  5117   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ true, vector_len);
  5256   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5118   emit_int8(0x1A);
  5257   emit_int8(0x1A);
  5119   emit_operand(dst, src);
  5258   emit_operand(dst, src);
       
  5259   // 0x00 - insert into lower 256 bits
  5120   // 0x01 - insert into upper 128 bits
  5260   // 0x01 - insert into upper 128 bits
  5121   emit_int8(0x01);
  5261   emit_int8(value & 0x01);
  5122 }
  5262 }
  5123 
  5263 
  5124 void Assembler::vinsertf32x4h(XMMRegister dst, XMMRegister nds, XMMRegister src, int value) {
  5264 void Assembler::vinsertf32x4h(XMMRegister dst, XMMRegister nds, XMMRegister src, int value) {
  5125   assert(VM_Version::supports_evex(), "");
  5265   assert(VM_Version::supports_evex(), "");
  5126   int vector_len = AVX_512bit;
  5266   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  5127   int src_enc = src->encoding();
  5267   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  5128   int dst_enc = dst->encoding();
  5268   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5129   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  5130   int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
       
  5131                                      /* vex_w */ false, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
       
  5132   emit_int8(0x18);
  5269   emit_int8(0x18);
  5133   emit_int8((unsigned char)(0xC0 | encode));
  5270   emit_int8((unsigned char)(0xC0 | encode));
  5134   // 0x00 - insert into q0 128 bits (0..127)
  5271   // 0x00 - insert into q0 128 bits (0..127)
  5135   // 0x01 - insert into q1 128 bits (128..255)
  5272   // 0x01 - insert into q1 128 bits (128..255)
  5136   // 0x02 - insert into q2 128 bits (256..383)
  5273   // 0x02 - insert into q2 128 bits (256..383)
  5137   // 0x03 - insert into q3 128 bits (384..511)
  5274   // 0x03 - insert into q3 128 bits (384..511)
  5138   emit_int8(value & 0x3);
  5275   emit_int8(value & 0x3);
  5139 }
  5276 }
  5140 
  5277 
  5141 void Assembler::vinsertf32x4h(XMMRegister dst, Address src, int value) {
  5278 void Assembler::vinsertf32x4h(XMMRegister dst, Address src, int value) {
  5142   assert(VM_Version::supports_evex(), "");
  5279   assert(VM_Version::supports_avx(), "");
  5143   _tuple_type = EVEX_T4;
       
  5144   _input_size_in_bits = EVEX_32bit;
       
  5145   InstructionMark im(this);
       
  5146   int vector_len = AVX_512bit;
       
  5147   assert(dst != xnoreg, "sanity");
  5280   assert(dst != xnoreg, "sanity");
  5148   int dst_enc = dst->encoding();
  5281   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
       
  5282   InstructionMark im(this);
       
  5283   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
       
  5284   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
  5149   // swap src<->dst for encoding
  5285   // swap src<->dst for encoding
  5150   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
  5286   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5151   emit_int8(0x18);
  5287   emit_int8(0x18);
  5152   emit_operand(dst, src);
  5288   emit_operand(dst, src);
  5153   // 0x00 - insert into q0 128 bits (0..127)
  5289   // 0x00 - insert into q0 128 bits (0..127)
  5154   // 0x01 - insert into q1 128 bits (128..255)
  5290   // 0x01 - insert into q1 128 bits (128..255)
  5155   // 0x02 - insert into q2 128 bits (256..383)
  5291   // 0x02 - insert into q2 128 bits (256..383)
  5157   emit_int8(value & 0x3);
  5293   emit_int8(value & 0x3);
  5158 }
  5294 }
  5159 
  5295 
  5160 void Assembler::vinsertf128h(XMMRegister dst, Address src) {
  5296 void Assembler::vinsertf128h(XMMRegister dst, Address src) {
  5161   assert(VM_Version::supports_avx(), "");
  5297   assert(VM_Version::supports_avx(), "");
  5162   int vector_len = AVX_256bit;
       
  5163   if (VM_Version::supports_evex()) {
       
  5164     _tuple_type = EVEX_T4;
       
  5165     _input_size_in_bits = EVEX_32bit;
       
  5166     vector_len = AVX_512bit;
       
  5167   }
       
  5168   InstructionMark im(this);
       
  5169   assert(dst != xnoreg, "sanity");
  5298   assert(dst != xnoreg, "sanity");
  5170   int dst_enc = dst->encoding();
  5299   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
       
  5300   InstructionMark im(this);
       
  5301   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
       
  5302   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
  5171   // swap src<->dst for encoding
  5303   // swap src<->dst for encoding
  5172   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
  5304   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5173   emit_int8(0x18);
  5305   emit_int8(0x18);
  5174   emit_operand(dst, src);
  5306   emit_operand(dst, src);
  5175   // 0x01 - insert into upper 128 bits
  5307   // 0x01 - insert into upper 128 bits
  5176   emit_int8(0x01);
  5308   emit_int8(0x01);
  5177 }
  5309 }
  5178 
  5310 
  5179 void Assembler::vextractf128h(XMMRegister dst, XMMRegister src) {
  5311 void Assembler::vextractf128h(XMMRegister dst, XMMRegister src) {
  5180   assert(VM_Version::supports_avx(), "");
  5312   assert(VM_Version::supports_avx(), "");
  5181   int vector_len = AVX_256bit;
  5313   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
  5182   if (VM_Version::supports_evex()) {
  5314   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  5183     vector_len = AVX_512bit;
  5315   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5184   }
       
  5185   int encode = vex_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A);
       
  5186   emit_int8(0x19);
  5316   emit_int8(0x19);
  5187   emit_int8((unsigned char)(0xC0 | encode));
  5317   emit_int8((unsigned char)(0xC0 | encode));
  5188   // 0x00 - insert into lower 128 bits
  5318   // 0x00 - insert into lower 128 bits
  5189   // 0x01 - insert into upper 128 bits
  5319   // 0x01 - insert into upper 128 bits
  5190   emit_int8(0x01);
  5320   emit_int8(0x01);
  5191 }
  5321 }
  5192 
  5322 
  5193 void Assembler::vextractf128h(Address dst, XMMRegister src) {
  5323 void Assembler::vextractf128h(Address dst, XMMRegister src) {
  5194   assert(VM_Version::supports_avx(), "");
  5324   assert(VM_Version::supports_avx(), "");
  5195   int vector_len = AVX_256bit;
       
  5196   if (VM_Version::supports_evex()) {
       
  5197     _tuple_type = EVEX_T4;
       
  5198     _input_size_in_bits = EVEX_32bit;
       
  5199     vector_len = AVX_512bit;
       
  5200   }
       
  5201   InstructionMark im(this);
       
  5202   assert(src != xnoreg, "sanity");
  5325   assert(src != xnoreg, "sanity");
  5203   int src_enc = src->encoding();
  5326   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
  5204   vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
  5327   InstructionMark im(this);
       
  5328   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
       
  5329   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
       
  5330   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5205   emit_int8(0x19);
  5331   emit_int8(0x19);
  5206   emit_operand(src, dst);
  5332   emit_operand(src, dst);
  5207   // 0x01 - extract from upper 128 bits
  5333   // 0x01 - extract from upper 128 bits
  5208   emit_int8(0x01);
  5334   emit_int8(0x01);
  5209 }
  5335 }
  5210 
  5336 
  5211 void Assembler::vinserti128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  5337 void Assembler::vinserti128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  5212   assert(VM_Version::supports_avx2(), "");
  5338   assert(VM_Version::supports_avx2(), "");
  5213   int vector_len = AVX_256bit;
  5339   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
  5214   if (VM_Version::supports_evex()) {
  5340   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  5215     vector_len = AVX_512bit;
  5341   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  5216   }
  5342   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5217   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A);
       
  5218   emit_int8(0x38);
  5343   emit_int8(0x38);
  5219   emit_int8((unsigned char)(0xC0 | encode));
  5344   emit_int8((unsigned char)(0xC0 | encode));
  5220   // 0x00 - insert into lower 128 bits
  5345   // 0x00 - insert into lower 128 bits
  5221   // 0x01 - insert into upper 128 bits
  5346   // 0x01 - insert into upper 128 bits
  5222   emit_int8(0x01);
  5347   emit_int8(0x01);
  5223 }
  5348 }
  5224 
  5349 
  5225 void Assembler::vinserti64x4h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
  5350 void Assembler::vinserti64x4h(XMMRegister dst, XMMRegister nds, XMMRegister src, int value) {
  5226   assert(VM_Version::supports_evex(), "");
  5351   assert(VM_Version::supports_evex(), "");
  5227   int vector_len = AVX_512bit;
  5352   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  5228   int src_enc = src->encoding();
  5353   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  5229   int dst_enc = dst->encoding();
  5354   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5230   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  5231   int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
       
  5232                                      /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_reg_mask */ false);
       
  5233   emit_int8(0x38);
  5355   emit_int8(0x38);
  5234   emit_int8((unsigned char)(0xC0 | encode));
  5356   emit_int8((unsigned char)(0xC0 | encode));
  5235   // 0x00 - insert into lower 256 bits
  5357   // 0x00 - insert into lower 256 bits
  5236   // 0x01 - insert into upper 256 bits
  5358   // 0x01 - insert into upper 256 bits
  5237   emit_int8(0x01);
  5359   emit_int8(value & 0x01);
  5238 }
  5360 }
  5239 
  5361 
  5240 void Assembler::vinserti128h(XMMRegister dst, Address src) {
  5362 void Assembler::vinserti128h(XMMRegister dst, Address src) {
  5241   assert(VM_Version::supports_avx2(), "");
  5363   assert(VM_Version::supports_avx2(), "");
  5242   int vector_len = AVX_256bit;
       
  5243   if (VM_Version::supports_evex()) {
       
  5244     _tuple_type = EVEX_T4;
       
  5245     _input_size_in_bits = EVEX_32bit;
       
  5246     vector_len = AVX_512bit;
       
  5247   }
       
  5248   InstructionMark im(this);
       
  5249   assert(dst != xnoreg, "sanity");
  5364   assert(dst != xnoreg, "sanity");
  5250   int dst_enc = dst->encoding();
  5365   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
       
  5366   InstructionMark im(this);
       
  5367   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
       
  5368   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
  5251   // swap src<->dst for encoding
  5369   // swap src<->dst for encoding
  5252   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
  5370   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5253   emit_int8(0x38);
  5371   emit_int8(0x38);
  5254   emit_operand(dst, src);
  5372   emit_operand(dst, src);
  5255   // 0x01 - insert into upper 128 bits
  5373   // 0x01 - insert into upper 128 bits
  5256   emit_int8(0x01);
  5374   emit_int8(0x01);
  5257 }
  5375 }
  5258 
  5376 
  5259 void Assembler::vextracti128h(XMMRegister dst, XMMRegister src) {
  5377 void Assembler::vextracti128h(XMMRegister dst, XMMRegister src) {
  5260   assert(VM_Version::supports_avx(), "");
  5378   assert(VM_Version::supports_avx(), "");
  5261   int vector_len = AVX_256bit;
  5379   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
  5262   if (VM_Version::supports_evex()) {
  5380   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  5263     vector_len = AVX_512bit;
  5381   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5264   }
       
  5265   int encode = vex_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A);
       
  5266   emit_int8(0x39);
  5382   emit_int8(0x39);
  5267   emit_int8((unsigned char)(0xC0 | encode));
  5383   emit_int8((unsigned char)(0xC0 | encode));
  5268   // 0x00 - insert into lower 128 bits
  5384   // 0x00 - insert into lower 128 bits
  5269   // 0x01 - insert into upper 128 bits
  5385   // 0x01 - insert into upper 128 bits
  5270   emit_int8(0x01);
  5386   emit_int8(0x01);
  5271 }
  5387 }
  5272 
  5388 
  5273 void Assembler::vextracti128h(Address dst, XMMRegister src) {
  5389 void Assembler::vextracti128h(Address dst, XMMRegister src) {
  5274   assert(VM_Version::supports_avx2(), "");
  5390   assert(VM_Version::supports_avx2(), "");
  5275   int vector_len = AVX_256bit;
       
  5276   if (VM_Version::supports_evex()) {
       
  5277     _tuple_type = EVEX_T4;
       
  5278     _input_size_in_bits = EVEX_32bit;
       
  5279     vector_len = AVX_512bit;
       
  5280   }
       
  5281   InstructionMark im(this);
       
  5282   assert(src != xnoreg, "sanity");
  5391   assert(src != xnoreg, "sanity");
  5283   int src_enc = src->encoding();
  5392   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
  5284   vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
  5393   InstructionMark im(this);
       
  5394   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
       
  5395   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
       
  5396   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5285   emit_int8(0x39);
  5397   emit_int8(0x39);
  5286   emit_operand(src, dst);
  5398   emit_operand(src, dst);
  5287   // 0x01 - extract from upper 128 bits
  5399   // 0x01 - extract from upper 128 bits
  5288   emit_int8(0x01);
  5400   emit_int8(0x01);
  5289 }
  5401 }
  5290 
  5402 
  5291 void Assembler::vextracti64x4h(XMMRegister dst, XMMRegister src) {
  5403 void Assembler::vextracti64x4h(XMMRegister dst, XMMRegister src, int value) {
  5292   assert(VM_Version::supports_evex(), "");
  5404   assert(VM_Version::supports_evex(), "");
  5293   int vector_len = AVX_512bit;
  5405   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  5294   int src_enc = src->encoding();
  5406   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5295   int dst_enc = dst->encoding();
       
  5296   int encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
       
  5297                                      /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
       
  5298   emit_int8(0x3B);
  5407   emit_int8(0x3B);
  5299   emit_int8((unsigned char)(0xC0 | encode));
  5408   emit_int8((unsigned char)(0xC0 | encode));
       
  5409   // 0x00 - extract from lower 256 bits
  5300   // 0x01 - extract from upper 256 bits
  5410   // 0x01 - extract from upper 256 bits
  5301   emit_int8(0x01);
  5411   emit_int8(value & 0x01);
  5302 }
  5412 }
  5303 
  5413 
  5304 void Assembler::vextracti64x2h(XMMRegister dst, XMMRegister src, int value) {
  5414 void Assembler::vextracti64x2h(XMMRegister dst, XMMRegister src, int value) {
  5305   assert(VM_Version::supports_evex(), "");
  5415   assert(VM_Version::supports_evex(), "");
  5306   int vector_len = AVX_512bit;
  5416   InstructionAttr attributes(AVX_512bit, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  5307   int src_enc = src->encoding();
  5417   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5308   int dst_enc = dst->encoding();
       
  5309   int encode;
       
  5310   if (VM_Version::supports_avx512dq()) {
       
  5311     encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
       
  5312                                    /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
       
  5313   } else {
       
  5314     encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
       
  5315                                    /* vex_w */ false, vector_len, /* legacy_mode */ true, /* no_mask_reg */ false);
       
  5316   }
       
  5317   emit_int8(0x39);
  5418   emit_int8(0x39);
  5318   emit_int8((unsigned char)(0xC0 | encode));
  5419   emit_int8((unsigned char)(0xC0 | encode));
  5319   // 0x01 - extract from bits 255:128
  5420   // 0x01 - extract from bits 255:128
  5320   // 0x02 - extract from bits 383:256
  5421   // 0x02 - extract from bits 383:256
  5321   // 0x03 - extract from bits 511:384
  5422   // 0x03 - extract from bits 511:384
  5322   emit_int8(value & 0x3);
  5423   emit_int8(value & 0x3);
  5323 }
  5424 }
  5324 
  5425 
  5325 void Assembler::vextractf64x4h(XMMRegister dst, XMMRegister src) {
  5426 void Assembler::vextractf64x4h(XMMRegister dst, XMMRegister src, int value) {
  5326   assert(VM_Version::supports_evex(), "");
  5427   assert(VM_Version::supports_evex(), "");
  5327   int vector_len = AVX_512bit;
  5428   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  5328   int src_enc = src->encoding();
  5429   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5329   int dst_enc = dst->encoding();
       
  5330   int encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
       
  5331                                      /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
       
  5332   emit_int8(0x1B);
  5430   emit_int8(0x1B);
  5333   emit_int8((unsigned char)(0xC0 | encode));
  5431   emit_int8((unsigned char)(0xC0 | encode));
       
  5432   // 0x00 - extract from lower 256 bits
  5334   // 0x01 - extract from upper 256 bits
  5433   // 0x01 - extract from upper 256 bits
  5335   emit_int8(0x01);
  5434   emit_int8(value & 0x1);
  5336 }
  5435 }
  5337 
  5436 
  5338 void Assembler::vextractf64x4h(Address dst, XMMRegister src) {
  5437 void Assembler::vextractf64x4h(Address dst, XMMRegister src, int value) {
  5339   assert(VM_Version::supports_evex(), "");
  5438   assert(VM_Version::supports_evex(), "");
  5340   _tuple_type = EVEX_T4;
       
  5341   _input_size_in_bits = EVEX_64bit;
       
  5342   InstructionMark im(this);
       
  5343   int vector_len = AVX_512bit;
       
  5344   assert(src != xnoreg, "sanity");
  5439   assert(src != xnoreg, "sanity");
  5345   int src_enc = src->encoding();
  5440   InstructionMark im(this);
  5346   vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
  5441   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  5347              /* vex_w */ true, vector_len);
  5442   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
       
  5443   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5348   emit_int8(0x1B);
  5444   emit_int8(0x1B);
  5349   emit_operand(src, dst);
  5445   emit_operand(src, dst);
       
  5446   // 0x00 - extract from lower 256 bits
  5350   // 0x01 - extract from upper 256 bits
  5447   // 0x01 - extract from upper 256 bits
  5351   emit_int8(0x01);
  5448   emit_int8(value & 0x01);
  5352 }
  5449 }
  5353 
  5450 
  5354 void Assembler::vextractf32x4h(XMMRegister dst, XMMRegister src, int value) {
  5451 void Assembler::vextractf32x4h(XMMRegister dst, XMMRegister src, int value) {
  5355   assert(VM_Version::supports_evex(), "");
  5452   assert(VM_Version::supports_avx(), "");
  5356   int vector_len = AVX_512bit;
  5453   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
  5357   int src_enc = src->encoding();
  5454   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  5358   int dst_enc = dst->encoding();
  5455   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5359   int encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
       
  5360                                      /* vex_w */ false, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
       
  5361   emit_int8(0x19);
  5456   emit_int8(0x19);
  5362   emit_int8((unsigned char)(0xC0 | encode));
  5457   emit_int8((unsigned char)(0xC0 | encode));
  5363   // 0x00 - extract from bits 127:0
  5458   // 0x00 - extract from bits 127:0
  5364   // 0x01 - extract from bits 255:128
  5459   // 0x01 - extract from bits 255:128
  5365   // 0x02 - extract from bits 383:256
  5460   // 0x02 - extract from bits 383:256
  5367   emit_int8(value & 0x3);
  5462   emit_int8(value & 0x3);
  5368 }
  5463 }
  5369 
  5464 
  5370 void Assembler::vextractf32x4h(Address dst, XMMRegister src, int value) {
  5465 void Assembler::vextractf32x4h(Address dst, XMMRegister src, int value) {
  5371   assert(VM_Version::supports_evex(), "");
  5466   assert(VM_Version::supports_evex(), "");
  5372   _tuple_type = EVEX_T4;
       
  5373   _input_size_in_bits = EVEX_32bit;
       
  5374   InstructionMark im(this);
       
  5375   int vector_len = AVX_512bit;
       
  5376   assert(src != xnoreg, "sanity");
  5467   assert(src != xnoreg, "sanity");
  5377   int src_enc = src->encoding();
  5468   InstructionMark im(this);
  5378   vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
  5469   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
       
  5470   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
       
  5471   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5379   emit_int8(0x19);
  5472   emit_int8(0x19);
  5380   emit_operand(src, dst);
  5473   emit_operand(src, dst);
  5381   // 0x00 - extract from bits 127:0
  5474   // 0x00 - extract from bits 127:0
  5382   // 0x01 - extract from bits 255:128
  5475   // 0x01 - extract from bits 255:128
  5383   // 0x02 - extract from bits 383:256
  5476   // 0x02 - extract from bits 383:256
  5385   emit_int8(value & 0x3);
  5478   emit_int8(value & 0x3);
  5386 }
  5479 }
  5387 
  5480 
  5388 void Assembler::vextractf64x2h(XMMRegister dst, XMMRegister src, int value) {
  5481 void Assembler::vextractf64x2h(XMMRegister dst, XMMRegister src, int value) {
  5389   assert(VM_Version::supports_evex(), "");
  5482   assert(VM_Version::supports_evex(), "");
  5390   int vector_len = AVX_512bit;
  5483   InstructionAttr attributes(AVX_512bit, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
  5391   int src_enc = src->encoding();
  5484   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5392   int dst_enc = dst->encoding();
       
  5393   int encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
       
  5394                                      /* vex_w */ !_legacy_mode_dq, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
       
  5395   emit_int8(0x19);
  5485   emit_int8(0x19);
  5396   emit_int8((unsigned char)(0xC0 | encode));
  5486   emit_int8((unsigned char)(0xC0 | encode));
  5397   // 0x01 - extract from bits 255:128
  5487   // 0x01 - extract from bits 255:128
  5398   // 0x02 - extract from bits 383:256
  5488   // 0x02 - extract from bits 383:256
  5399   // 0x03 - extract from bits 511:384
  5489   // 0x03 - extract from bits 511:384
  5400   emit_int8(value & 0x3);
  5490   emit_int8(value & 0x3);
  5401 }
  5491 }
  5402 
  5492 
  5403 // duplicate 4-bytes integer data from src into 8 locations in dest
  5493 // duplicate 4-bytes integer data from src into 8 locations in dest
  5404 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
  5494 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
  5405   _instruction_uses_vl = true;
  5495   assert(VM_Version::supports_avx2(), "");
  5406   assert(UseAVX > 1, "");
  5496   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  5407   int vector_len = AVX_256bit;
  5497   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  5408   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);
       
  5409   emit_int8(0x58);
  5498   emit_int8(0x58);
  5410   emit_int8((unsigned char)(0xC0 | encode));
  5499   emit_int8((unsigned char)(0xC0 | encode));
  5411 }
  5500 }
  5412 
  5501 
  5413 // duplicate 2-bytes integer data from src into 16 locations in dest
  5502 // duplicate 2-bytes integer data from src into 16 locations in dest
  5414 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
  5503 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
  5415   assert(VM_Version::supports_avx2(), "");
  5504   assert(VM_Version::supports_avx2(), "");
  5416   bool vector_len = AVX_256bit;
  5505   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
  5417   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66,
  5506   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  5418                                      vector_len, VEX_OPCODE_0F_38, false);
       
  5419   emit_int8(0x79);
  5507   emit_int8(0x79);
  5420   emit_int8((unsigned char)(0xC0 | encode));
  5508   emit_int8((unsigned char)(0xC0 | encode));
  5421 }
  5509 }
  5422 
  5510 
  5423 // duplicate 1-byte integer data from src into 16||32|64 locations in dest : requires AVX512BW and AVX512VL
  5511 // duplicate 1-byte integer data from src into 16||32|64 locations in dest : requires AVX512BW and AVX512VL
  5424 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
  5512 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
  5425   _instruction_uses_vl = true;
  5513   assert(VM_Version::supports_evex(), "");
  5426   assert(UseAVX > 1, "");
  5514   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  5427   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);
  5515   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  5428   emit_int8(0x78);
  5516   emit_int8(0x78);
  5429   emit_int8((unsigned char)(0xC0 | encode));
  5517   emit_int8((unsigned char)(0xC0 | encode));
  5430 }
  5518 }
  5431 
  5519 
  5432 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
  5520 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
  5433   _instruction_uses_vl = true;
  5521   assert(VM_Version::supports_evex(), "");
  5434   assert(UseAVX > 1, "");
       
  5435   _tuple_type = EVEX_T1S;
       
  5436   _input_size_in_bits = EVEX_8bit;
       
  5437   InstructionMark im(this);
       
  5438   assert(dst != xnoreg, "sanity");
  5522   assert(dst != xnoreg, "sanity");
  5439   int dst_enc = dst->encoding();
  5523   InstructionMark im(this);
       
  5524   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  5525   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
  5440   // swap src<->dst for encoding
  5526   // swap src<->dst for encoding
  5441   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
  5527   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  5442   emit_int8(0x78);
  5528   emit_int8(0x78);
  5443   emit_operand(dst, src);
  5529   emit_operand(dst, src);
  5444 }
  5530 }
  5445 
  5531 
  5446 // duplicate 2-byte integer data from src into 8|16||32 locations in dest : requires AVX512BW and AVX512VL
  5532 // duplicate 2-byte integer data from src into 8|16||32 locations in dest : requires AVX512BW and AVX512VL
  5447 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
  5533 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
  5448   _instruction_uses_vl = true;
  5534   assert(VM_Version::supports_evex(), "");
  5449   assert(UseAVX > 1, "");
  5535   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  5450   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);
  5536   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  5451   emit_int8(0x79);
  5537   emit_int8(0x79);
  5452   emit_int8((unsigned char)(0xC0 | encode));
  5538   emit_int8((unsigned char)(0xC0 | encode));
  5453 }
  5539 }
  5454 
  5540 
  5455 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
  5541 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
  5456   _instruction_uses_vl = true;
  5542   assert(VM_Version::supports_evex(), "");
  5457   assert(UseAVX > 1, "");
       
  5458   _tuple_type = EVEX_T1S;
       
  5459   _input_size_in_bits = EVEX_16bit;
       
  5460   InstructionMark im(this);
       
  5461   assert(dst != xnoreg, "sanity");
  5543   assert(dst != xnoreg, "sanity");
  5462   int dst_enc = dst->encoding();
  5544   InstructionMark im(this);
       
  5545   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  5546   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
  5463   // swap src<->dst for encoding
  5547   // swap src<->dst for encoding
  5464   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
  5548   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  5465   emit_int8(0x79);
  5549   emit_int8(0x79);
  5466   emit_operand(dst, src);
  5550   emit_operand(dst, src);
  5467 }
  5551 }
  5468 
  5552 
  5469 // duplicate 4-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
  5553 // duplicate 4-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
  5470 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
  5554 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
  5471   _instruction_uses_vl = true;
  5555   assert(VM_Version::supports_evex(), "");
  5472   assert(UseAVX > 1, "");
  5556   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  5473   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);
  5557   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  5474   emit_int8(0x58);
  5558   emit_int8(0x58);
  5475   emit_int8((unsigned char)(0xC0 | encode));
  5559   emit_int8((unsigned char)(0xC0 | encode));
  5476 }
  5560 }
  5477 
  5561 
  5478 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
  5562 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
  5479   _instruction_uses_vl = true;
  5563   assert(VM_Version::supports_evex(), "");
  5480   assert(UseAVX > 1, "");
       
  5481   _tuple_type = EVEX_T1S;
       
  5482   _input_size_in_bits = EVEX_32bit;
       
  5483   InstructionMark im(this);
       
  5484   assert(dst != xnoreg, "sanity");
  5564   assert(dst != xnoreg, "sanity");
  5485   int dst_enc = dst->encoding();
  5565   InstructionMark im(this);
       
  5566   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  5567   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
  5486   // swap src<->dst for encoding
  5568   // swap src<->dst for encoding
  5487   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
  5569   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  5488   emit_int8(0x58);
  5570   emit_int8(0x58);
  5489   emit_operand(dst, src);
  5571   emit_operand(dst, src);
  5490 }
  5572 }
  5491 
  5573 
  5492 // duplicate 8-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
  5574 // duplicate 8-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
  5493 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
  5575 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
  5494   _instruction_uses_vl = true;
  5576   assert(VM_Version::supports_evex(), "");
  5495   assert(UseAVX > 1, "");
  5577   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  5496   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
  5578   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  5497                                      /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
       
  5498   emit_int8(0x59);
  5579   emit_int8(0x59);
  5499   emit_int8((unsigned char)(0xC0 | encode));
  5580   emit_int8((unsigned char)(0xC0 | encode));
  5500 }
  5581 }
  5501 
  5582 
  5502 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
  5583 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
  5503   _instruction_uses_vl = true;
  5584   assert(VM_Version::supports_evex(), "");
  5504   assert(UseAVX > 1, "");
       
  5505   _tuple_type = EVEX_T1S;
       
  5506   _input_size_in_bits = EVEX_64bit;
       
  5507   InstructionMark im(this);
       
  5508   assert(dst != xnoreg, "sanity");
  5585   assert(dst != xnoreg, "sanity");
  5509   int dst_enc = dst->encoding();
  5586   InstructionMark im(this);
       
  5587   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  5588   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  5510   // swap src<->dst for encoding
  5589   // swap src<->dst for encoding
  5511   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ true, vector_len);
  5590   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  5512   emit_int8(0x59);
  5591   emit_int8(0x59);
  5513   emit_operand(dst, src);
  5592   emit_operand(dst, src);
  5514 }
  5593 }
  5515 
  5594 
  5516 // duplicate single precision fp from src into 4|8|16 locations in dest : requires AVX512VL
  5595 // duplicate single precision fp from src into 4|8|16 locations in dest : requires AVX512VL
  5517 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
  5596 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
  5518   _instruction_uses_vl = true;
  5597   assert(VM_Version::supports_evex(), "");
  5519   assert(UseAVX > 1, "");
  5598   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  5520   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
  5599   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  5521                                      /* vex_w */ false, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
       
  5522   emit_int8(0x18);
  5600   emit_int8(0x18);
  5523   emit_int8((unsigned char)(0xC0 | encode));
  5601   emit_int8((unsigned char)(0xC0 | encode));
  5524 }
  5602 }
  5525 
  5603 
  5526 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
  5604 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
  5527   assert(UseAVX > 1, "");
  5605   assert(VM_Version::supports_evex(), "");
  5528   _tuple_type = EVEX_T1S;
       
  5529   _input_size_in_bits = EVEX_32bit;
       
  5530   InstructionMark im(this);
       
  5531   assert(dst != xnoreg, "sanity");
  5606   assert(dst != xnoreg, "sanity");
  5532   int dst_enc = dst->encoding();
  5607   InstructionMark im(this);
       
  5608   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  5609   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
  5533   // swap src<->dst for encoding
  5610   // swap src<->dst for encoding
  5534   vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
  5611   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  5535   emit_int8(0x18);
  5612   emit_int8(0x18);
  5536   emit_operand(dst, src);
  5613   emit_operand(dst, src);
  5537 }
  5614 }
  5538 
  5615 
  5539 // duplicate double precision fp from src into 2|4|8 locations in dest : requires AVX512VL
  5616 // duplicate double precision fp from src into 2|4|8 locations in dest : requires AVX512VL
  5540 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
  5617 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
  5541   _instruction_uses_vl = true;
  5618   assert(VM_Version::supports_evex(), "");
  5542   assert(UseAVX > 1, "");
  5619   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  5543   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
  5620   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  5544                                      /*vex_w */ true, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
       
  5545   emit_int8(0x19);
  5621   emit_int8(0x19);
  5546   emit_int8((unsigned char)(0xC0 | encode));
  5622   emit_int8((unsigned char)(0xC0 | encode));
  5547 }
  5623 }
  5548 
  5624 
  5549 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
  5625 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
  5550   _instruction_uses_vl = true;
  5626   assert(VM_Version::supports_evex(), "");
  5551   assert(UseAVX > 1, "");
       
  5552   _tuple_type = EVEX_T1S;
       
  5553   _input_size_in_bits = EVEX_64bit;
       
  5554   InstructionMark im(this);
       
  5555   assert(dst != xnoreg, "sanity");
  5627   assert(dst != xnoreg, "sanity");
  5556   int dst_enc = dst->encoding();
  5628   InstructionMark im(this);
       
  5629   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
       
  5630   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  5557   // swap src<->dst for encoding
  5631   // swap src<->dst for encoding
  5558   vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ true, vector_len);
  5632   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  5559   emit_int8(0x19);
  5633   emit_int8(0x19);
  5560   emit_operand(dst, src);
  5634   emit_operand(dst, src);
  5561 }
  5635 }
  5562 
  5636 
  5563 // duplicate 1-byte integer data from src into 16||32|64 locations in dest : requires AVX512BW and AVX512VL
  5637 // duplicate 1-byte integer data from src into 16||32|64 locations in dest : requires AVX512BW and AVX512VL
  5564 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
  5638 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
  5565   _instruction_uses_vl = true;
       
  5566   assert(VM_Version::supports_evex(), "");
  5639   assert(VM_Version::supports_evex(), "");
  5567   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
  5640   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  5568                                      /*vex_w */ false, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
  5641   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  5569   emit_int8(0x7A);
  5642   emit_int8(0x7A);
  5570   emit_int8((unsigned char)(0xC0 | encode));
  5643   emit_int8((unsigned char)(0xC0 | encode));
  5571 }
  5644 }
  5572 
  5645 
  5573 // duplicate 2-byte integer data from src into 8|16||32 locations in dest : requires AVX512BW and AVX512VL
  5646 // duplicate 2-byte integer data from src into 8|16||32 locations in dest : requires AVX512BW and AVX512VL
  5574 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
  5647 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
  5575   _instruction_uses_vl = true;
       
  5576   assert(VM_Version::supports_evex(), "");
  5648   assert(VM_Version::supports_evex(), "");
  5577   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
  5649   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  5578                                      /* vex_w */ false, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
  5650   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  5579   emit_int8(0x7B);
  5651   emit_int8(0x7B);
  5580   emit_int8((unsigned char)(0xC0 | encode));
  5652   emit_int8((unsigned char)(0xC0 | encode));
  5581 }
  5653 }
  5582 
  5654 
  5583 // duplicate 4-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
  5655 // duplicate 4-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
  5584 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
  5656 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
  5585   _instruction_uses_vl = true;
       
  5586   assert(VM_Version::supports_evex(), "");
  5657   assert(VM_Version::supports_evex(), "");
  5587   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
  5658   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  5588                                      /* vex_w */ false, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
  5659   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  5589   emit_int8(0x7C);
  5660   emit_int8(0x7C);
  5590   emit_int8((unsigned char)(0xC0 | encode));
  5661   emit_int8((unsigned char)(0xC0 | encode));
  5591 }
  5662 }
  5592 
  5663 
  5593 // duplicate 8-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
  5664 // duplicate 8-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
  5594 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
  5665 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
  5595   _instruction_uses_vl = true;
       
  5596   assert(VM_Version::supports_evex(), "");
  5666   assert(VM_Version::supports_evex(), "");
  5597   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
  5667   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
  5598                                      /* vex_w */ true, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
  5668   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  5599   emit_int8(0x7C);
  5669   emit_int8(0x7C);
  5600   emit_int8((unsigned char)(0xC0 | encode));
  5670   emit_int8((unsigned char)(0xC0 | encode));
  5601 }
  5671 }
  5602 
  5672 
  5603 // Carry-Less Multiplication Quadword
  5673 // Carry-Less Multiplication Quadword
  5604 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
  5674 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
  5605   assert(VM_Version::supports_clmul(), "");
  5675   assert(VM_Version::supports_clmul(), "");
  5606   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
  5676   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  5607                                       VEX_OPCODE_0F_3A, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
  5677   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5608   emit_int8(0x44);
  5678   emit_int8(0x44);
  5609   emit_int8((unsigned char)(0xC0 | encode));
  5679   emit_int8((unsigned char)(0xC0 | encode));
  5610   emit_int8((unsigned char)mask);
  5680   emit_int8((unsigned char)mask);
  5611 }
  5681 }
  5612 
  5682 
  5613 // Carry-Less Multiplication Quadword
  5683 // Carry-Less Multiplication Quadword
  5614 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
  5684 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
  5615   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
  5685   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
  5616   int vector_len = AVX_128bit;
  5686   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  5617   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A, /* legacy_mode */ true);
  5687   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  5688   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  5618   emit_int8(0x44);
  5689   emit_int8(0x44);
  5619   emit_int8((unsigned char)(0xC0 | encode));
  5690   emit_int8((unsigned char)(0xC0 | encode));
  5620   emit_int8((unsigned char)mask);
  5691   emit_int8((unsigned char)mask);
  5621 }
  5692 }
  5622 
  5693 
  5623 void Assembler::vzeroupper() {
  5694 void Assembler::vzeroupper() {
  5624   assert(VM_Version::supports_avx(), "");
  5695   assert(VM_Version::supports_avx(), "");
  5625   if (UseAVX < 3)
  5696   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  5626   {
  5697   (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
  5627     (void)vex_prefix_and_encode(xmm0, xmm0, xmm0, VEX_SIMD_NONE);
  5698   emit_int8(0x77);
  5628     emit_int8(0x77);
       
  5629   }
       
  5630 }
  5699 }
  5631 
  5700 
  5632 
  5701 
  5633 #ifndef _LP64
  5702 #ifndef _LP64
  5634 // 32bit only pieces of the assembler
  5703 // 32bit only pieces of the assembler
  6128 
  6197 
  6129 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
  6198 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
  6130   if (pre > 0) {
  6199   if (pre > 0) {
  6131     emit_int8(simd_pre[pre]);
  6200     emit_int8(simd_pre[pre]);
  6132   }
  6201   }
  6133   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) :
  6202   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
  6134                           prefix_and_encode(dst_enc, src_enc);
       
  6135   if (opc > 0) {
  6203   if (opc > 0) {
  6136     emit_int8(0x0F);
  6204     emit_int8(0x0F);
  6137     int opc2 = simd_opc[opc];
  6205     int opc2 = simd_opc[opc];
  6138     if (opc2 > 0) {
  6206     if (opc2 > 0) {
  6139       emit_int8(opc2);
  6207       emit_int8(opc2);
  6141   }
  6209   }
  6142   return encode;
  6210   return encode;
  6143 }
  6211 }
  6144 
  6212 
  6145 
  6213 
  6146 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, bool vex_w, int nds_enc, VexSimdPrefix pre, VexOpcode opc, int vector_len) {
  6214 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
       
  6215   int vector_len = _attributes->get_vector_len();
       
  6216   bool vex_w = _attributes->is_rex_vex_w();
  6147   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
  6217   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
  6148     prefix(VEX_3bytes);
  6218     prefix(VEX_3bytes);
  6149 
  6219 
  6150     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
  6220     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
  6151     byte1 = (~byte1) & 0xE0;
  6221     byte1 = (~byte1) & 0xE0;
  6165     emit_int8(byte1);
  6235     emit_int8(byte1);
  6166   }
  6236   }
  6167 }
  6237 }
  6168 
  6238 
  6169 // This is a 4 byte encoding
  6239 // This is a 4 byte encoding
  6170 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool vex_w, bool evex_r, bool evex_v,
  6240 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
  6171                             int nds_enc, VexSimdPrefix pre, VexOpcode opc,
       
  6172                             bool is_extended_context, bool is_merge_context,
       
  6173                             int vector_len, bool no_mask_reg ){
       
  6174   // EVEX 0x62 prefix
  6241   // EVEX 0x62 prefix
  6175   prefix(EVEX_4bytes);
  6242   prefix(EVEX_4bytes);
  6176   _evex_encoding = (vex_w ? VEX_W : 0) | (evex_r ? EVEX_Rb : 0);
  6243   bool vex_w = _attributes->is_rex_vex_w();
       
  6244   int evex_encoding = (vex_w ? VEX_W : 0);
       
  6245   // EVEX.b is not currently used for broadcast of single element or data rounding modes
       
  6246   _attributes->set_evex_encoding(evex_encoding);
  6177 
  6247 
  6178   // P0: byte 2, initialized to RXBR`00mm
  6248   // P0: byte 2, initialized to RXBR`00mm
  6179   // instead of not'd
  6249   // instead of not'd
  6180   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
  6250   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
  6181   byte2 = (~byte2) & 0xF0;
  6251   byte2 = (~byte2) & 0xF0;
  6193   // of form {66, F3, F2}
  6263   // of form {66, F3, F2}
  6194   byte3 |= pre;
  6264   byte3 |= pre;
  6195   emit_int8(byte3);
  6265   emit_int8(byte3);
  6196 
  6266 
  6197   // P2: byte 4 as zL'Lbv'aaa
  6267   // P2: byte 4 as zL'Lbv'aaa
  6198   int byte4 = (no_mask_reg) ? 0 : 1; // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
  6268   int byte4 = (_attributes->is_no_reg_mask()) ? 0 : 1; // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
  6199   // EVEX.v` for extending EVEX.vvvv or VIDX
  6269   // EVEX.v` for extending EVEX.vvvv or VIDX
  6200   byte4 |= (evex_v ? 0: EVEX_V);
  6270   byte4 |= (evex_v ? 0: EVEX_V);
  6201   // third EXEC.b for broadcast actions
  6271   // third EXEC.b for broadcast actions
  6202   byte4 |= (is_extended_context ? EVEX_Rb : 0);
  6272   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
  6203   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
  6273   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
  6204   byte4 |= ((vector_len) & 0x3) << 5;
  6274   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
  6205   // last is EVEX.z for zero/merge actions
  6275   // last is EVEX.z for zero/merge actions
  6206   byte4 |= (is_merge_context ? EVEX_Z : 0);
  6276   byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
  6207   emit_int8(byte4);
  6277   emit_int8(byte4);
  6208 }
  6278 }
  6209 
  6279 
  6210 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre,
  6280 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
  6211                            VexOpcode opc, bool vex_w, int vector_len, bool legacy_mode, bool no_mask_reg) {
       
  6212   bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
  6281   bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
  6213   bool vex_b = adr.base_needs_rex();
  6282   bool vex_b = adr.base_needs_rex();
  6214   bool vex_x = adr.index_needs_rex();
  6283   bool vex_x = adr.index_needs_rex();
  6215   _avx_vector_len = vector_len;
  6284   set_attributes(attributes);
       
  6285   attributes->set_current_assembler(this);
  6216 
  6286 
  6217   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
  6287   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
  6218   if (_legacy_mode_vl && _instruction_uses_vl) {
  6288   if ((UseAVX > 2) && _legacy_mode_vl && attributes->uses_vl()) {
  6219     switch (vector_len) {
  6289     switch (attributes->get_vector_len()) {
  6220     case AVX_128bit:
  6290     case AVX_128bit:
  6221     case AVX_256bit:
  6291     case AVX_256bit:
  6222       legacy_mode = true;
  6292       attributes->set_is_legacy_mode();
  6223       break;
  6293       break;
  6224     }
  6294     }
  6225   }
  6295   }
  6226 
  6296 
  6227   if ((UseAVX > 2) && (legacy_mode == false))
  6297   if ((UseAVX > 2) && !attributes->is_legacy_mode())
  6228   {
  6298   {
  6229     bool evex_r = (xreg_enc >= 16);
  6299     bool evex_r = (xreg_enc >= 16);
  6230     bool evex_v = (nds_enc >= 16);
  6300     bool evex_v = (nds_enc >= 16);
  6231     _is_evex_instruction = true;
  6301     attributes->set_is_evex_instruction();
  6232     evex_prefix(vex_r, vex_b, vex_x, vex_w, evex_r, evex_v, nds_enc, pre, opc, false, false, vector_len, no_mask_reg);
  6302     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
  6233   } else {
  6303   } else {
  6234     vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector_len);
  6304     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
  6235   }
  6305   }
  6236   _instruction_uses_vl = false;
  6306 }
  6237 }
  6307 
  6238 
  6308 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
  6239 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc,
       
  6240                                      bool vex_w, int vector_len, bool legacy_mode, bool no_mask_reg ) {
       
  6241   bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
  6309   bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
  6242   bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
  6310   bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
  6243   bool vex_x = false;
  6311   bool vex_x = false;
  6244   _avx_vector_len = vector_len;
  6312   set_attributes(attributes);
       
  6313   attributes->set_current_assembler(this);
  6245 
  6314 
  6246   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
  6315   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
  6247   if (_legacy_mode_vl && _instruction_uses_vl) {
  6316   if ((UseAVX > 2) && _legacy_mode_vl && attributes->uses_vl()) {
  6248     switch (vector_len) {
  6317     switch (attributes->get_vector_len()) {
  6249     case AVX_128bit:
  6318     case AVX_128bit:
  6250     case AVX_256bit:
  6319     case AVX_256bit:
  6251       legacy_mode = true;
  6320       if ((dst_enc >= 16) | (nds_enc >= 16) | (src_enc >= 16)) {
       
  6321         // up propagate arithmetic instructions to meet RA requirements
       
  6322         attributes->set_vector_len(AVX_512bit);
       
  6323       } else {
       
  6324         attributes->set_is_legacy_mode();
       
  6325       }
  6252       break;
  6326       break;
  6253     }
  6327     }
  6254   }
  6328   }
  6255 
  6329 
  6256   if ((UseAVX > 2) && (legacy_mode == false))
  6330   if ((UseAVX > 2) && !attributes->is_legacy_mode())
  6257   {
  6331   {
  6258     bool evex_r = (dst_enc >= 16);
  6332     bool evex_r = (dst_enc >= 16);
  6259     bool evex_v = (nds_enc >= 16);
  6333     bool evex_v = (nds_enc >= 16);
  6260     // can use vex_x as bank extender on rm encoding
  6334     // can use vex_x as bank extender on rm encoding
  6261     vex_x = (src_enc >= 16);
  6335     vex_x = (src_enc >= 16);
  6262     evex_prefix(vex_r, vex_b, vex_x, vex_w, evex_r, evex_v, nds_enc, pre, opc, false, false, vector_len, no_mask_reg);
  6336     attributes->set_is_evex_instruction();
       
  6337     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
  6263   } else {
  6338   } else {
  6264     vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector_len);
  6339     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
  6265   }
  6340   }
  6266 
       
  6267   _instruction_uses_vl = false;
       
  6268 
  6341 
  6269   // return modrm byte components for operands
  6342   // return modrm byte components for operands
  6270   return (((dst_enc & 7) << 3) | (src_enc & 7));
  6343   return (((dst_enc & 7) << 3) | (src_enc & 7));
  6271 }
  6344 }
  6272 
  6345 
  6273 
  6346 
  6274 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
  6347 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
  6275                             bool no_mask_reg, VexOpcode opc, bool rex_w, int vector_len, bool legacy_mode) {
  6348                             VexOpcode opc, InstructionAttr *attributes) {
  6276   if (UseAVX > 0) {
  6349   if (UseAVX > 0) {
  6277     int xreg_enc = xreg->encoding();
  6350     int xreg_enc = xreg->encoding();
  6278     int  nds_enc = nds->is_valid() ? nds->encoding() : 0;
  6351     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  6279     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, rex_w, vector_len, legacy_mode, no_mask_reg);
  6352     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
  6280   } else {
  6353   } else {
  6281     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
  6354     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
  6282     rex_prefix(adr, xreg, pre, opc, rex_w);
  6355     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
  6283   }
  6356   }
  6284 }
  6357 }
  6285 
  6358 
  6286 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
  6359 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
  6287                                       bool no_mask_reg, VexOpcode opc, bool rex_w, int vector_len, bool legacy_mode) {
  6360                                       VexOpcode opc, InstructionAttr *attributes) {
  6288   int dst_enc = dst->encoding();
  6361   int dst_enc = dst->encoding();
  6289   int src_enc = src->encoding();
  6362   int src_enc = src->encoding();
  6290   if (UseAVX > 0) {
  6363   if (UseAVX > 0) {
  6291     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  6364     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  6292     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, rex_w, vector_len, legacy_mode, no_mask_reg);
  6365     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
  6293   } else {
  6366   } else {
  6294     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
  6367     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
  6295     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, rex_w);
  6368     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
  6296   }
  6369   }
  6297 }
  6370 }
  6298 
  6371 
  6299 int Assembler::kreg_prefix_and_encode(KRegister dst, KRegister nds, KRegister src, VexSimdPrefix pre,
  6372 int Assembler::kreg_prefix_and_encode(KRegister dst, KRegister nds, KRegister src, VexSimdPrefix pre,
  6300                                       bool no_mask_reg, VexOpcode opc, bool rex_w, int vector_len) {
  6373                                       VexOpcode opc, InstructionAttr *attributes) {
  6301   int dst_enc = dst->encoding();
  6374   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  6302   int src_enc = src->encoding();
  6375   return vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), pre, opc, attributes);
  6303   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  6304   return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, rex_w, vector_len, true, no_mask_reg);
       
  6305 }
  6376 }
  6306 
  6377 
  6307 int Assembler::kreg_prefix_and_encode(KRegister dst, KRegister nds, Register src, VexSimdPrefix pre,
  6378 int Assembler::kreg_prefix_and_encode(KRegister dst, KRegister nds, Register src, VexSimdPrefix pre,
  6308                                       bool no_mask_reg, VexOpcode opc, bool rex_w, int vector_len) {
  6379                                       VexOpcode opc, InstructionAttr *attributes) {
  6309   int dst_enc = dst->encoding();
  6380   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
  6310   int src_enc = src->encoding();
  6381   return vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), pre, opc, attributes);
  6311   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  6312   return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, rex_w, vector_len, true, no_mask_reg);
       
  6313 }
       
  6314 
       
  6315 void Assembler::emit_simd_arith(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre, bool no_mask_reg, bool legacy_mode) {
       
  6316   InstructionMark im(this);
       
  6317   simd_prefix(dst, dst, src, pre, no_mask_reg, VEX_OPCODE_0F, false, AVX_128bit, legacy_mode);
       
  6318   emit_int8(opcode);
       
  6319   emit_operand(dst, src);
       
  6320 }
       
  6321 
       
  6322 void Assembler::emit_simd_arith_q(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre, bool no_mask_reg) {
       
  6323   InstructionMark im(this);
       
  6324   simd_prefix_q(dst, dst, src, pre, no_mask_reg);
       
  6325   emit_int8(opcode);
       
  6326   emit_operand(dst, src);
       
  6327 }
       
  6328 
       
  6329 void Assembler::emit_simd_arith(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre, bool no_mask_reg, bool legacy_mode) {
       
  6330   int encode = simd_prefix_and_encode(dst, dst, src, pre, no_mask_reg, VEX_OPCODE_0F, false, AVX_128bit, legacy_mode);
       
  6331   emit_int8(opcode);
       
  6332   emit_int8((unsigned char)(0xC0 | encode));
       
  6333 }
       
  6334 
       
  6335 void Assembler::emit_simd_arith_q(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre, bool no_mask_reg) {
       
  6336   int encode = simd_prefix_and_encode(dst, dst, src, pre, no_mask_reg, VEX_OPCODE_0F, true, AVX_128bit);
       
  6337   emit_int8(opcode);
       
  6338   emit_int8((unsigned char)(0xC0 | encode));
       
  6339 }
       
  6340 
       
  6341 // Versions with no second source register (non-destructive source).
       
  6342 void Assembler::emit_simd_arith_nonds(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre, bool opNoRegMask) {
       
  6343   InstructionMark im(this);
       
  6344   simd_prefix(dst, xnoreg, src, pre, opNoRegMask);
       
  6345   emit_int8(opcode);
       
  6346   emit_operand(dst, src);
       
  6347 }
       
  6348 
       
  6349 void Assembler::emit_simd_arith_nonds_q(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre, bool opNoRegMask) {
       
  6350   InstructionMark im(this);
       
  6351   simd_prefix_q(dst, xnoreg, src, pre, opNoRegMask);
       
  6352   emit_int8(opcode);
       
  6353   emit_operand(dst, src);
       
  6354 }
       
  6355 
       
  6356 void Assembler::emit_simd_arith_nonds(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre, bool no_mask_reg, bool legacy_mode) {
       
  6357   int encode = simd_prefix_and_encode(dst, xnoreg, src, pre, no_mask_reg, VEX_OPCODE_0F, false, AVX_128bit, legacy_mode);
       
  6358   emit_int8(opcode);
       
  6359   emit_int8((unsigned char)(0xC0 | encode));
       
  6360 }
       
  6361 
       
  6362 void Assembler::emit_simd_arith_nonds_q(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre, bool no_mask_reg) {
       
  6363   int encode = simd_prefix_and_encode(dst, xnoreg, src, pre, no_mask_reg, VEX_OPCODE_0F, true);
       
  6364   emit_int8(opcode);
       
  6365   emit_int8((unsigned char)(0xC0 | encode));
       
  6366 }
       
  6367 
       
  6368 // 3-operands AVX instructions
       
  6369 void Assembler::emit_vex_arith(int opcode, XMMRegister dst, XMMRegister nds, Address src,
       
  6370                                VexSimdPrefix pre, int vector_len, bool no_mask_reg, bool legacy_mode) {
       
  6371   InstructionMark im(this);
       
  6372   vex_prefix(dst, nds, src, pre, vector_len, no_mask_reg, legacy_mode);
       
  6373   emit_int8(opcode);
       
  6374   emit_operand(dst, src);
       
  6375 }
       
  6376 
       
  6377 void Assembler::emit_vex_arith_q(int opcode, XMMRegister dst, XMMRegister nds,
       
  6378                                  Address src, VexSimdPrefix pre, int vector_len, bool no_mask_reg) {
       
  6379   InstructionMark im(this);
       
  6380   vex_prefix_q(dst, nds, src, pre, vector_len, no_mask_reg);
       
  6381   emit_int8(opcode);
       
  6382   emit_operand(dst, src);
       
  6383 }
       
  6384 
       
  6385 void Assembler::emit_vex_arith(int opcode, XMMRegister dst, XMMRegister nds, XMMRegister src,
       
  6386                                VexSimdPrefix pre, int vector_len, bool no_mask_reg, bool legacy_mode) {
       
  6387   int encode = vex_prefix_and_encode(dst, nds, src, pre, vector_len, VEX_OPCODE_0F, legacy_mode, no_mask_reg);
       
  6388   emit_int8(opcode);
       
  6389   emit_int8((unsigned char)(0xC0 | encode));
       
  6390 }
       
  6391 
       
  6392 void Assembler::emit_vex_arith_q(int opcode, XMMRegister dst, XMMRegister nds, XMMRegister src,
       
  6393                                  VexSimdPrefix pre, int vector_len, bool no_mask_reg) {
       
  6394   int src_enc = src->encoding();
       
  6395   int dst_enc = dst->encoding();
       
  6396   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  6397   int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, VEX_OPCODE_0F, true, vector_len, false, no_mask_reg);
       
  6398   emit_int8(opcode);
       
  6399   emit_int8((unsigned char)(0xC0 | encode));
       
  6400 }
  6382 }
  6401 
  6383 
  6402 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
  6384 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
  6403   assert(VM_Version::supports_avx(), "");
  6385   assert(VM_Version::supports_avx(), "");
  6404   assert(!VM_Version::supports_evex(), "");
  6386   assert(!VM_Version::supports_evex(), "");
  6405   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F, /* no_mask_reg */ false);
  6387   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  6388   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  6406   emit_int8((unsigned char)0xC2);
  6389   emit_int8((unsigned char)0xC2);
  6407   emit_int8((unsigned char)(0xC0 | encode));
  6390   emit_int8((unsigned char)(0xC0 | encode));
  6408   emit_int8((unsigned char)(0xF & cop));
  6391   emit_int8((unsigned char)(0xF & cop));
  6409 }
  6392 }
  6410 
  6393 
  6411 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
  6394 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
  6412   assert(VM_Version::supports_avx(), "");
  6395   assert(VM_Version::supports_avx(), "");
  6413   assert(!VM_Version::supports_evex(), "");
  6396   assert(!VM_Version::supports_evex(), "");
  6414   int encode = vex_prefix_and_encode(dst, nds, src1, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A, /* no_mask_reg */ false);
  6397   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  6398   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
       
  6399   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
  6415   emit_int8((unsigned char)0x4B);
  6400   emit_int8((unsigned char)0x4B);
  6416   emit_int8((unsigned char)(0xC0 | encode));
  6401   emit_int8((unsigned char)(0xC0 | encode));
  6417   int src2_enc = src2->encoding();
  6402   int src2_enc = src2->encoding();
  6418   emit_int8((unsigned char)(0xF0 & src2_enc<<4));
  6403   emit_int8((unsigned char)(0xF0 & src2_enc<<4));
  6419 }
  6404 }
  6428 
  6413 
  6429 void Assembler::lea(Register dst, Address src) {
  6414 void Assembler::lea(Register dst, Address src) {
  6430   leal(dst, src);
  6415   leal(dst, src);
  6431 }
  6416 }
  6432 
  6417 
  6433 void Assembler::mov_literal32(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
  6418 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
  6434   InstructionMark im(this);
  6419   InstructionMark im(this);
  6435   emit_int8((unsigned char)0xC7);
  6420   emit_int8((unsigned char)0xC7);
  6436   emit_operand(rax, dst);
  6421   emit_operand(rax, dst);
  6437   emit_data((int)imm32, rspec, 0);
  6422   emit_data((int)imm32, rspec, 0);
  6438 }
  6423 }
  6946   emit_arith(0x23, 0xC0, dst, src);
  6931   emit_arith(0x23, 0xC0, dst, src);
  6947 }
  6932 }
  6948 
  6933 
  6949 void Assembler::andnq(Register dst, Register src1, Register src2) {
  6934 void Assembler::andnq(Register dst, Register src1, Register src2) {
  6950   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  6935   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  6951   int encode = vex_prefix_0F38_and_encode_q_legacy(dst, src1, src2);
  6936   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  6937   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
  6952   emit_int8((unsigned char)0xF2);
  6938   emit_int8((unsigned char)0xF2);
  6953   emit_int8((unsigned char)(0xC0 | encode));
  6939   emit_int8((unsigned char)(0xC0 | encode));
  6954 }
  6940 }
  6955 
  6941 
  6956 void Assembler::andnq(Register dst, Register src1, Address src2) {
  6942 void Assembler::andnq(Register dst, Register src1, Address src2) {
  6957   InstructionMark im(this);
       
  6958   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  6943   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  6959   vex_prefix_0F38_q_legacy(dst, src1, src2);
  6944   InstructionMark im(this);
       
  6945   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  6946   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
  6960   emit_int8((unsigned char)0xF2);
  6947   emit_int8((unsigned char)0xF2);
  6961   emit_operand(dst, src2);
  6948   emit_operand(dst, src2);
  6962 }
  6949 }
  6963 
  6950 
  6964 void Assembler::bsfq(Register dst, Register src) {
  6951 void Assembler::bsfq(Register dst, Register src) {
  6981   emit_int8((unsigned char)(0xC8 | encode));
  6968   emit_int8((unsigned char)(0xC8 | encode));
  6982 }
  6969 }
  6983 
  6970 
  6984 void Assembler::blsiq(Register dst, Register src) {
  6971 void Assembler::blsiq(Register dst, Register src) {
  6985   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  6972   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  6986   int encode = vex_prefix_0F38_and_encode_q_legacy(rbx, dst, src);
  6973   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  6974   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
  6987   emit_int8((unsigned char)0xF3);
  6975   emit_int8((unsigned char)0xF3);
  6988   emit_int8((unsigned char)(0xC0 | encode));
  6976   emit_int8((unsigned char)(0xC0 | encode));
  6989 }
  6977 }
  6990 
  6978 
  6991 void Assembler::blsiq(Register dst, Address src) {
  6979 void Assembler::blsiq(Register dst, Address src) {
  6992   InstructionMark im(this);
       
  6993   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  6980   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  6994   vex_prefix_0F38_q_legacy(rbx, dst, src);
  6981   InstructionMark im(this);
       
  6982   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  6983   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
  6995   emit_int8((unsigned char)0xF3);
  6984   emit_int8((unsigned char)0xF3);
  6996   emit_operand(rbx, src);
  6985   emit_operand(rbx, src);
  6997 }
  6986 }
  6998 
  6987 
  6999 void Assembler::blsmskq(Register dst, Register src) {
  6988 void Assembler::blsmskq(Register dst, Register src) {
  7000   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  6989   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  7001   int encode = vex_prefix_0F38_and_encode_q_legacy(rdx, dst, src);
  6990   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  6991   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
  7002   emit_int8((unsigned char)0xF3);
  6992   emit_int8((unsigned char)0xF3);
  7003   emit_int8((unsigned char)(0xC0 | encode));
  6993   emit_int8((unsigned char)(0xC0 | encode));
  7004 }
  6994 }
  7005 
  6995 
  7006 void Assembler::blsmskq(Register dst, Address src) {
  6996 void Assembler::blsmskq(Register dst, Address src) {
  7007   InstructionMark im(this);
       
  7008   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  6997   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  7009   vex_prefix_0F38_q_legacy(rdx, dst, src);
  6998   InstructionMark im(this);
       
  6999   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  7000   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
  7010   emit_int8((unsigned char)0xF3);
  7001   emit_int8((unsigned char)0xF3);
  7011   emit_operand(rdx, src);
  7002   emit_operand(rdx, src);
  7012 }
  7003 }
  7013 
  7004 
  7014 void Assembler::blsrq(Register dst, Register src) {
  7005 void Assembler::blsrq(Register dst, Register src) {
  7015   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  7006   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  7016   int encode = vex_prefix_0F38_and_encode_q_legacy(rcx, dst, src);
  7007   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  7008   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
  7017   emit_int8((unsigned char)0xF3);
  7009   emit_int8((unsigned char)0xF3);
  7018   emit_int8((unsigned char)(0xC0 | encode));
  7010   emit_int8((unsigned char)(0xC0 | encode));
  7019 }
  7011 }
  7020 
  7012 
  7021 void Assembler::blsrq(Register dst, Address src) {
  7013 void Assembler::blsrq(Register dst, Address src) {
  7022   InstructionMark im(this);
       
  7023   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  7014   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
  7024   vex_prefix_0F38_q_legacy(rcx, dst, src);
  7015   InstructionMark im(this);
       
  7016   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
       
  7017   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
  7025   emit_int8((unsigned char)0xF3);
  7018   emit_int8((unsigned char)0xF3);
  7026   emit_operand(rcx, src);
  7019   emit_operand(rcx, src);
  7027 }
  7020 }
  7028 
  7021 
  7029 void Assembler::cdqq() {
  7022 void Assembler::cdqq() {
  7093   emit_operand(reg, adr);
  7086   emit_operand(reg, adr);
  7094 }
  7087 }
  7095 
  7088 
  7096 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
  7089 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
  7097   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  7090   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  7098   int encode = simd_prefix_and_encode_q(dst, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);
  7091   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  7092   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  7099   emit_int8(0x2A);
  7093   emit_int8(0x2A);
  7100   emit_int8((unsigned char)(0xC0 | encode));
  7094   emit_int8((unsigned char)(0xC0 | encode));
  7101 }
  7095 }
  7102 
  7096 
  7103 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
  7097 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
  7104   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  7098   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  7105   if (VM_Version::supports_evex()) {
  7099   InstructionMark im(this);
  7106     _tuple_type = EVEX_T1S;
  7100   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  7107     _input_size_in_bits = EVEX_32bit;
  7101   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  7108   }
  7102   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  7109   InstructionMark im(this);
       
  7110   simd_prefix_q(dst, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);
       
  7111   emit_int8(0x2A);
  7103   emit_int8(0x2A);
  7112   emit_operand(dst, src);
  7104   emit_operand(dst, src);
  7113 }
  7105 }
  7114 
  7106 
  7115 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
  7107 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
  7116   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  7108   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  7117   if (VM_Version::supports_evex()) {
  7109   InstructionMark im(this);
  7118     _tuple_type = EVEX_T1S;
  7110   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  7119     _input_size_in_bits = EVEX_32bit;
  7111   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
  7120   }
  7112   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  7121   InstructionMark im(this);
       
  7122   simd_prefix_q(dst, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
       
  7123   emit_int8(0x2A);
  7113   emit_int8(0x2A);
  7124   emit_operand(dst, src);
  7114   emit_operand(dst, src);
  7125 }
  7115 }
  7126 
  7116 
  7127 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
  7117 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
  7128   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  7118   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  7129   int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, /* no_mask_reg */ true);
  7119   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  7120   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
  7130   emit_int8(0x2C);
  7121   emit_int8(0x2C);
  7131   emit_int8((unsigned char)(0xC0 | encode));
  7122   emit_int8((unsigned char)(0xC0 | encode));
  7132 }
  7123 }
  7133 
  7124 
  7134 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
  7125 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
  7135   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  7126   NOT_LP64(assert(VM_Version::supports_sse(), ""));
  7136   int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, /* no_mask_reg */ true);
  7127   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  7128   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
  7137   emit_int8(0x2C);
  7129   emit_int8(0x2C);
  7138   emit_int8((unsigned char)(0xC0 | encode));
  7130   emit_int8((unsigned char)(0xC0 | encode));
  7139 }
  7131 }
  7140 
  7132 
  7141 void Assembler::decl(Register dst) {
  7133 void Assembler::decl(Register dst) {
  7314 }
  7306 }
  7315 
  7307 
  7316 void Assembler::movdq(XMMRegister dst, Register src) {
  7308 void Assembler::movdq(XMMRegister dst, Register src) {
  7317   // table D-1 says MMX/SSE2
  7309   // table D-1 says MMX/SSE2
  7318   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  7310   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  7319   int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
  7311   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
       
  7312   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  7320   emit_int8(0x6E);
  7313   emit_int8(0x6E);
  7321   emit_int8((unsigned char)(0xC0 | encode));
  7314   emit_int8((unsigned char)(0xC0 | encode));
  7322 }
  7315 }
  7323 
  7316 
  7324 void Assembler::movdq(Register dst, XMMRegister src) {
  7317 void Assembler::movdq(Register dst, XMMRegister src) {
  7325   // table D-1 says MMX/SSE2
  7318   // table D-1 says MMX/SSE2
  7326   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
  7319   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
       
  7320   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  7327   // swap src/dst to get correct prefix
  7321   // swap src/dst to get correct prefix
  7328   int encode = simd_prefix_and_encode_q(src, dst, VEX_SIMD_66, /* no_mask_reg */ true);
  7322   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
  7329   emit_int8(0x7E);
  7323   emit_int8(0x7E);
  7330   emit_int8((unsigned char)(0xC0 | encode));
  7324   emit_int8((unsigned char)(0xC0 | encode));
  7331 }
  7325 }
  7332 
  7326 
  7333 void Assembler::movq(Register dst, Register src) {
  7327 void Assembler::movq(Register dst, Register src) {
  7456   emit_int8((unsigned char)(0xE0 | encode));
  7450   emit_int8((unsigned char)(0xE0 | encode));
  7457 }
  7451 }
  7458 
  7452 
  7459 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
  7453 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
  7460   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
  7454   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
  7461   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38,
  7455   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  7462                                     /* vex_w */ true, AVX_128bit, /* legacy_mode */ true, /* no_mask_reg */ false);
  7456   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
  7463   emit_int8((unsigned char)0xF6);
  7457   emit_int8((unsigned char)0xF6);
  7464   emit_int8((unsigned char)(0xC0 | encode));
  7458   emit_int8((unsigned char)(0xC0 | encode));
  7465 }
  7459 }
  7466 
  7460 
  7467 void Assembler::negq(Register dst) {
  7461 void Assembler::negq(Register dst) {
  7619   }
  7613   }
  7620 }
  7614 }
  7621 
  7615 
  7622 void Assembler::rorxq(Register dst, Register src, int imm8) {
  7616 void Assembler::rorxq(Register dst, Register src, int imm8) {
  7623   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
  7617   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
  7624   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A,
  7618   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
  7625                                      /* vex_w */ true, AVX_128bit, /* legacy_mode */ true, /* no_mask_reg */ false);
  7619   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
  7626   emit_int8((unsigned char)0xF0);
  7620   emit_int8((unsigned char)0xF0);
  7627   emit_int8((unsigned char)(0xC0 | encode));
  7621   emit_int8((unsigned char)(0xC0 | encode));
  7628   emit_int8(imm8);
  7622   emit_int8(imm8);
  7629 }
  7623 }
  7630 
  7624