8215483: Off heap memory accesses should be vectorized
Reviewed-by: neliasso, kvn
--- a/src/hotspot/cpu/aarch64/aarch64.ad Tue Feb 05 10:56:12 2019 +0100
+++ b/src/hotspot/cpu/aarch64/aarch64.ad Tue Dec 18 09:29:39 2018 +0100
@@ -2146,9 +2146,9 @@
return false;
}
-// x86 supports misaligned vectors store/load.
+// aarch64 supports misaligned vectors store/load.
const bool Matcher::misaligned_vectors_ok() {
- return !AlignVector; // can be changed by flag
+ return true;
}
// false => size gets scaled to BytesPerLong, ok.
--- a/src/hotspot/cpu/ppc/ppc.ad Tue Feb 05 10:56:12 2019 +0100
+++ b/src/hotspot/cpu/ppc/ppc.ad Tue Dec 18 09:29:39 2018 +0100
@@ -2338,9 +2338,10 @@
return max_vector_size(bt); // Same as max.
}
-// PPC doesn't support misaligned vectors store/load.
+// PPC implementation uses VSX load/store instructions (if
+// SuperwordUseVSX) which support 4 byte but not arbitrary alignment
const bool Matcher::misaligned_vectors_ok() {
- return !AlignVector; // can be changed by flag
+ return false;
}
// PPC AES support not yet implemented
--- a/src/hotspot/cpu/x86/x86.ad Tue Feb 05 10:56:12 2019 +0100
+++ b/src/hotspot/cpu/x86/x86.ad Tue Dec 18 09:29:39 2018 +0100
@@ -1585,7 +1585,7 @@
// x86 supports misaligned vectors store/load.
const bool Matcher::misaligned_vectors_ok() {
- return !AlignVector; // can be changed by flag
+ return true;
}
// x86 AES instructions are compatible with SunJCE expanded
--- a/src/hotspot/share/opto/superword.cpp Tue Feb 05 10:56:12 2019 +0100
+++ b/src/hotspot/share/opto/superword.cpp Tue Dec 18 09:29:39 2018 +0100
@@ -613,7 +613,7 @@
// alignment is set and vectors will be aligned.
bool create_pack = true;
if (memory_alignment(mem_ref, best_iv_adjustment) == 0 || _do_vector_loop) {
- if (!Matcher::misaligned_vectors_ok()) {
+ if (!Matcher::misaligned_vectors_ok() || AlignVector) {
int vw = vector_width(mem_ref);
int vw_best = vector_width(best_align_to_mem_ref);
if (vw > vw_best) {
@@ -638,7 +638,7 @@
} else {
// Allow independent (different type) unaligned memory operations
// if HW supports them.
- if (!Matcher::misaligned_vectors_ok()) {
+ if (!Matcher::misaligned_vectors_ok() || AlignVector) {
create_pack = false;
} else {
// Check if packs of the same memory type but
@@ -3372,9 +3372,9 @@
_igvn.register_new_node_with_optimizer(e);
_phase->set_ctrl(e, pre_ctrl);
}
- if (vw > ObjectAlignmentInBytes) {
+ if (vw > ObjectAlignmentInBytes || align_to_ref_p.base()->is_top()) {
// incorporate base e +/- base && Mask >>> log2(elt)
- Node* xbase = new CastP2XNode(NULL, align_to_ref_p.base());
+ Node* xbase = new CastP2XNode(NULL, align_to_ref_p.adr());
_igvn.register_new_node_with_optimizer(xbase);
#ifdef _LP64
xbase = new ConvL2INode(xbase);
@@ -3566,8 +3566,8 @@
assert(!valid(), "base address is loop variant");
return;
}
- //unsafe reference could not be aligned appropriately without runtime checking
- if (base == NULL || base->bottom_type() == Type::TOP) {
+ // unsafe references require misaligned vector access support
+ if (base->is_top() && !Matcher::misaligned_vectors_ok()) {
assert(!valid(), "unsafe access");
return;
}
@@ -3591,6 +3591,16 @@
break; // stop looking at addp's
}
}
+ if (!invariant(adr)) {
+ assert(!valid(), "adr is loop variant");
+ return;
+ }
+
+ if (!base->is_top() && adr != base) {
+ assert(!valid(), "adr and base differ");
+ return;
+ }
+
NOT_PRODUCT(if(_slp->is_trace_alignment()) _tracer.restore_depth();)
NOT_PRODUCT(_tracer.ctor_6(mem);)