--- a/hotspot/make/bsd/makefiles/gcc.make Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/make/bsd/makefiles/gcc.make Thu Aug 21 19:06:21 2014 +0000
@@ -508,13 +508,9 @@
ifeq ($(USE_CLANG),)
# Enable bounds checking.
- # _FORTIFY_SOURCE appears in GCC 4.0+
ifeq "$(shell expr \( $(CC_VER_MAJOR) \> 3 \) )" "1"
- # compile time size bounds checks
- FASTDEBUG_CFLAGS += -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=1
-
- # and runtime size bounds checks and paranoid stack smashing checks.
- DEBUG_CFLAGS += -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -fstack-protector-all --param ssp-buffer-size=1
+ # stack smashing checks.
+ DEBUG_CFLAGS += -fstack-protector-all --param ssp-buffer-size=1
endif
endif
--- a/hotspot/make/linux/makefiles/gcc.make Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/make/linux/makefiles/gcc.make Thu Aug 21 19:06:21 2014 +0000
@@ -365,16 +365,13 @@
ifeq ($(USE_CLANG),)
# Enable bounds checking.
- # _FORTIFY_SOURCE appears in GCC 4.0+
ifeq "$(shell expr \( $(CC_VER_MAJOR) \> 3 \) )" "1"
- # compile time size bounds checks
- FASTDEBUG_CFLAGS += -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=1
-
- # and runtime size bounds checks and paranoid stack smashing checks.
- DEBUG_CFLAGS += -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -fstack-protector-all --param ssp-buffer-size=1
+ # stack smashing checks.
+ DEBUG_CFLAGS += -fstack-protector-all --param ssp-buffer-size=1
endif
endif
+
# If we are building HEADLESS, pass on to VM
# so it can set the java.awt.headless property
ifdef HEADLESS
--- a/hotspot/make/solaris/makefiles/gcc.make Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/make/solaris/makefiles/gcc.make Thu Aug 21 19:06:21 2014 +0000
@@ -240,11 +240,7 @@
endif
# Enable bounds checking.
-# _FORTIFY_SOURCE appears in GCC 4.0+
ifeq "$(shell expr \( $(CC_VER_MAJOR) \> 3 \) )" "1"
- # compile time size bounds checks
- FASTDEBUG_CFLAGS += -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=1
-
- # and runtime size bounds checks and paranoid stack smashing checks.
- DEBUG_CFLAGS += -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -fstack-protector-all --param ssp-buffer-size=1
+ # stack smashing checks.
+ DEBUG_CFLAGS += -fstack-protector-all --param ssp-buffer-size=1
endif
--- a/hotspot/src/os_cpu/solaris_sparc/vm/vm_version_solaris_sparc.cpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/os_cpu/solaris_sparc/vm/vm_version_solaris_sparc.cpp Thu Aug 21 19:06:21 2014 +0000
@@ -66,129 +66,92 @@
}
int VM_Version::platform_features(int features) {
- // getisax(2), SI_ARCHITECTURE_32, and SI_ARCHITECTURE_64 are
- // supported on Solaris 10 and later.
- if (os::Solaris::supports_getisax()) {
+ assert(os::Solaris::supports_getisax(), "getisax() must be available");
- // Check 32-bit architecture.
- do_sysinfo(SI_ARCHITECTURE_32, "sparc", &features, v8_instructions_m);
+ // Check 32-bit architecture.
+ do_sysinfo(SI_ARCHITECTURE_32, "sparc", &features, v8_instructions_m);
- // Check 64-bit architecture.
- do_sysinfo(SI_ARCHITECTURE_64, "sparcv9", &features, generic_v9_m);
+ // Check 64-bit architecture.
+ do_sysinfo(SI_ARCHITECTURE_64, "sparcv9", &features, generic_v9_m);
- // Extract valid instruction set extensions.
- uint_t avs[2];
- uint_t avn = os::Solaris::getisax(avs, 2);
- assert(avn <= 2, "should return two or less av's");
- uint_t av = avs[0];
+ // Extract valid instruction set extensions.
+ uint_t avs[2];
+ uint_t avn = os::Solaris::getisax(avs, 2);
+ assert(avn <= 2, "should return two or less av's");
+ uint_t av = avs[0];
#ifndef PRODUCT
- if (PrintMiscellaneous && Verbose) {
- tty->print("getisax(2) returned: " PTR32_FORMAT, av);
- if (avn > 1) {
- tty->print(", " PTR32_FORMAT, avs[1]);
- }
- tty->cr();
+ if (PrintMiscellaneous && Verbose) {
+ tty->print("getisax(2) returned: " PTR32_FORMAT, av);
+ if (avn > 1) {
+ tty->print(", " PTR32_FORMAT, avs[1]);
}
+ tty->cr();
+ }
#endif
- if (av & AV_SPARC_MUL32) features |= hardware_mul32_m;
- if (av & AV_SPARC_DIV32) features |= hardware_div32_m;
- if (av & AV_SPARC_FSMULD) features |= hardware_fsmuld_m;
- if (av & AV_SPARC_V8PLUS) features |= v9_instructions_m;
- if (av & AV_SPARC_POPC) features |= hardware_popc_m;
- if (av & AV_SPARC_VIS) features |= vis1_instructions_m;
- if (av & AV_SPARC_VIS2) features |= vis2_instructions_m;
- if (avn > 1) {
- uint_t av2 = avs[1];
+ if (av & AV_SPARC_MUL32) features |= hardware_mul32_m;
+ if (av & AV_SPARC_DIV32) features |= hardware_div32_m;
+ if (av & AV_SPARC_FSMULD) features |= hardware_fsmuld_m;
+ if (av & AV_SPARC_V8PLUS) features |= v9_instructions_m;
+ if (av & AV_SPARC_POPC) features |= hardware_popc_m;
+ if (av & AV_SPARC_VIS) features |= vis1_instructions_m;
+ if (av & AV_SPARC_VIS2) features |= vis2_instructions_m;
+ if (avn > 1) {
+ uint_t av2 = avs[1];
#ifndef AV2_SPARC_SPARC5
#define AV2_SPARC_SPARC5 0x00000008 /* The 29 new fp and sub instructions */
#endif
- if (av2 & AV2_SPARC_SPARC5) features |= sparc5_instructions_m;
- }
+ if (av2 & AV2_SPARC_SPARC5) features |= sparc5_instructions_m;
+ }
- // Next values are not defined before Solaris 10
- // but Solaris 8 is used for jdk6 update builds.
+ // We only build on Solaris 10 and up, but some of the values below
+ // are not defined on all versions of Solaris 10, so we define them,
+ // if necessary.
#ifndef AV_SPARC_ASI_BLK_INIT
#define AV_SPARC_ASI_BLK_INIT 0x0080 /* ASI_BLK_INIT_xxx ASI */
#endif
- if (av & AV_SPARC_ASI_BLK_INIT) features |= blk_init_instructions_m;
+ if (av & AV_SPARC_ASI_BLK_INIT) features |= blk_init_instructions_m;
#ifndef AV_SPARC_FMAF
#define AV_SPARC_FMAF 0x0100 /* Fused Multiply-Add */
#endif
- if (av & AV_SPARC_FMAF) features |= fmaf_instructions_m;
+ if (av & AV_SPARC_FMAF) features |= fmaf_instructions_m;
#ifndef AV_SPARC_FMAU
-#define AV_SPARC_FMAU 0x0200 /* Unfused Multiply-Add */
+#define AV_SPARC_FMAU 0x0200 /* Unfused Multiply-Add */
#endif
- if (av & AV_SPARC_FMAU) features |= fmau_instructions_m;
+ if (av & AV_SPARC_FMAU) features |= fmau_instructions_m;
#ifndef AV_SPARC_VIS3
-#define AV_SPARC_VIS3 0x0400 /* VIS3 instruction set extensions */
+#define AV_SPARC_VIS3 0x0400 /* VIS3 instruction set extensions */
#endif
- if (av & AV_SPARC_VIS3) features |= vis3_instructions_m;
+ if (av & AV_SPARC_VIS3) features |= vis3_instructions_m;
#ifndef AV_SPARC_CBCOND
#define AV_SPARC_CBCOND 0x10000000 /* compare and branch instrs supported */
#endif
- if (av & AV_SPARC_CBCOND) features |= cbcond_instructions_m;
+ if (av & AV_SPARC_CBCOND) features |= cbcond_instructions_m;
#ifndef AV_SPARC_AES
#define AV_SPARC_AES 0x00020000 /* aes instrs supported */
#endif
- if (av & AV_SPARC_AES) features |= aes_instructions_m;
+ if (av & AV_SPARC_AES) features |= aes_instructions_m;
#ifndef AV_SPARC_SHA1
#define AV_SPARC_SHA1 0x00400000 /* sha1 instruction supported */
#endif
- if (av & AV_SPARC_SHA1) features |= sha1_instruction_m;
+ if (av & AV_SPARC_SHA1) features |= sha1_instruction_m;
#ifndef AV_SPARC_SHA256
#define AV_SPARC_SHA256 0x00800000 /* sha256 instruction supported */
#endif
- if (av & AV_SPARC_SHA256) features |= sha256_instruction_m;
+ if (av & AV_SPARC_SHA256) features |= sha256_instruction_m;
#ifndef AV_SPARC_SHA512
#define AV_SPARC_SHA512 0x01000000 /* sha512 instruction supported */
#endif
- if (av & AV_SPARC_SHA512) features |= sha512_instruction_m;
-
- } else {
- // getisax(2) failed, use the old legacy code.
-#ifndef PRODUCT
- if (PrintMiscellaneous && Verbose)
- tty->print_cr("getisax(2) is not supported.");
-#endif
-
- char tmp;
- size_t bufsize = sysinfo(SI_ISALIST, &tmp, 1);
- char* buf = (char*) os::malloc(bufsize, mtInternal);
-
- if (buf != NULL) {
- if (sysinfo(SI_ISALIST, buf, bufsize) == bufsize) {
- // Figure out what kind of sparc we have
- char *sparc_string = strstr(buf, "sparc");
- if (sparc_string != NULL) { features |= v8_instructions_m;
- if (sparc_string[5] == 'v') {
- if (sparc_string[6] == '8') {
- if (sparc_string[7] == '-') { features |= hardware_mul32_m;
- features |= hardware_div32_m;
- } else if (sparc_string[7] == 'p') features |= generic_v9_m;
- else features |= generic_v8_m;
- } else if (sparc_string[6] == '9') features |= generic_v9_m;
- }
- }
-
- // Check for visualization instructions
- char *vis = strstr(buf, "vis");
- if (vis != NULL) { features |= vis1_instructions_m;
- if (vis[3] == '2') features |= vis2_instructions_m;
- }
- }
- os::free(buf);
- }
- }
+ if (av & AV_SPARC_SHA512) features |= sha512_instruction_m;
// Determine the machine type.
do_sysinfo(SI_MACHINE, "sun4v", &features, sun4v_m);
@@ -203,27 +166,7 @@
kstat_named_t* knm = (kstat_named_t *)ksp->ks_data;
for (int i = 0; i < ksp->ks_ndata; i++) {
if (strcmp((const char*)&(knm[i].name),"implementation") == 0) {
-#ifndef KSTAT_DATA_STRING
-#define KSTAT_DATA_STRING 9
-#endif
- if (knm[i].data_type == KSTAT_DATA_CHAR) {
- // VM is running on Solaris 8 which does not have value.str.
- implementation = &(knm[i].value.c[0]);
- } else if (knm[i].data_type == KSTAT_DATA_STRING) {
- // VM is running on Solaris 10.
-#ifndef KSTAT_NAMED_STR_PTR
- // Solaris 8 was used to build VM, define the structure it misses.
- struct str_t {
- union {
- char *ptr; /* NULL-term string */
- char __pad[8]; /* 64-bit padding */
- } addr;
- uint32_t len; /* # bytes for strlen + '\0' */
- };
-#define KSTAT_NAMED_STR_PTR(knptr) (( (str_t*)&((knptr)->value) )->addr.ptr)
-#endif
- implementation = KSTAT_NAMED_STR_PTR(&knm[i]);
- }
+ implementation = KSTAT_NAMED_STR_PTR(&knm[i]);
#ifndef PRODUCT
if (PrintMiscellaneous && Verbose) {
tty->print_cr("cpu_info.implementation: %s", implementation);
@@ -234,6 +177,7 @@
for (int i = 0; impl[i] != 0; i++)
impl[i] = (char)toupper((uint)impl[i]);
+
if (strstr(impl, "SPARC64") != NULL) {
features |= sparc64_family_m;
} else if (strstr(impl, "SPARC-M") != NULL) {
@@ -248,8 +192,10 @@
if (strstr(impl, "SPARC") == NULL) {
#ifndef PRODUCT
// kstat on Solaris 8 virtual machines (branded zones)
- // returns "(unsupported)" implementation.
- warning("kstat cpu_info implementation = '%s', should contain SPARC", impl);
+ // returns "(unsupported)" implementation. Solaris 8 is not
+ // supported anymore, but include this check to be on the
+ // safe side.
+ warning("kstat cpu_info implementation = '%s', assume generic SPARC", impl);
#endif
implementation = "SPARC";
}
--- a/hotspot/src/share/vm/c1/c1_Runtime1.cpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/c1/c1_Runtime1.cpp Thu Aug 21 19:06:21 2014 +0000
@@ -546,13 +546,18 @@
// normal bytecode execution.
thread->clear_exception_oop_and_pc();
+ Handle original_exception(thread, exception());
+
continuation = SharedRuntime::compute_compiled_exc_handler(nm, pc, exception, false, false);
// If an exception was thrown during exception dispatch, the exception oop may have changed
thread->set_exception_oop(exception());
thread->set_exception_pc(pc);
// the exception cache is used only by non-implicit exceptions
- if (continuation != NULL) {
+ // Update the exception cache only when there didn't happen
+ // another exception during the computation of the compiled
+ // exception handler.
+ if (continuation != NULL && original_exception() == exception()) {
nm->add_handler_for_exception_and_pc(exception, pc, continuation);
}
}
--- a/hotspot/src/share/vm/classfile/dictionary.cpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/classfile/dictionary.cpp Thu Aug 21 19:06:21 2014 +0000
@@ -130,15 +130,13 @@
}
-bool Dictionary::do_unloading() {
+void Dictionary::do_unloading() {
assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
- bool class_was_unloaded = false;
- int index = 0; // Defined here for portability! Do not move
// Remove unloadable entries and classes from system dictionary
// The placeholder array has been handled in always_strong_oops_do.
DictionaryEntry* probe = NULL;
- for (index = 0; index < table_size(); index++) {
+ for (int index = 0; index < table_size(); index++) {
for (DictionaryEntry** p = bucket_addr(index); *p != NULL; ) {
probe = *p;
Klass* e = probe->klass();
@@ -158,16 +156,8 @@
// Do we need to delete this system dictionary entry?
if (loader_data->is_unloading()) {
// If the loader is not live this entry should always be
- // removed (will never be looked up again). Note that this is
- // not the same as unloading the referred class.
- if (k_def_class_loader_data == loader_data) {
- // This is the defining entry, so the referred class is about
- // to be unloaded.
- class_was_unloaded = true;
- }
- // Also remove this system dictionary entry.
+ // removed (will never be looked up again).
purge_entry = true;
-
} else {
// The loader in this entry is alive. If the klass is dead,
// (determined by checking the defining class loader)
@@ -196,7 +186,6 @@
p = probe->next_addr();
}
}
- return class_was_unloaded;
}
void Dictionary::roots_oops_do(OopClosure* strong, OopClosure* weak) {
--- a/hotspot/src/share/vm/classfile/dictionary.hpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/classfile/dictionary.hpp Thu Aug 21 19:06:21 2014 +0000
@@ -109,9 +109,8 @@
return (loader_data->is_the_null_class_loader_data() || !ClassUnloading);
}
- // Unload (that is, break root links to) all unmarked classes and
- // loaders. Returns "true" iff something was unloaded.
- bool do_unloading();
+ // Unload (that is, break root links to) all unmarked classes and loaders.
+ void do_unloading();
// Protection domains
Klass* find(int index, unsigned int hash, Symbol* name,
--- a/hotspot/src/share/vm/classfile/systemDictionary.cpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/classfile/systemDictionary.cpp Thu Aug 21 19:06:21 2014 +0000
@@ -1693,10 +1693,9 @@
// Note: anonymous classes are not in the SD.
bool SystemDictionary::do_unloading(BoolObjectClosure* is_alive) {
// First, mark for unload all ClassLoaderData referencing a dead class loader.
- bool has_dead_loaders = ClassLoaderDataGraph::do_unloading(is_alive);
- bool unloading_occurred = false;
- if (has_dead_loaders) {
- unloading_occurred = dictionary()->do_unloading();
+ bool unloading_occurred = ClassLoaderDataGraph::do_unloading(is_alive);
+ if (unloading_occurred) {
+ dictionary()->do_unloading();
constraints()->purge_loader_constraints();
resolution_errors()->purge_resolution_errors();
}
--- a/hotspot/src/share/vm/opto/callnode.cpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/opto/callnode.cpp Thu Aug 21 19:06:21 2014 +0000
@@ -1815,3 +1815,57 @@
}
return result;
}
+
+ArrayCopyNode::ArrayCopyNode(Compile* C, bool alloc_tightly_coupled)
+ : CallNode(arraycopy_type(), NULL, TypeRawPtr::BOTTOM), _alloc_tightly_coupled(alloc_tightly_coupled), _kind(ArrayCopy) {
+ init_class_id(Class_ArrayCopy);
+ init_flags(Flag_is_macro);
+ C->add_macro_node(this);
+}
+
+uint ArrayCopyNode::size_of() const { return sizeof(*this); }
+
+ArrayCopyNode* ArrayCopyNode::make(GraphKit* kit, bool may_throw,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* length,
+ bool alloc_tightly_coupled,
+ Node* src_length, Node* dest_length,
+ Node* src_klass, Node* dest_klass) {
+
+ ArrayCopyNode* ac = new ArrayCopyNode(kit->C, alloc_tightly_coupled);
+ Node* prev_mem = kit->set_predefined_input_for_runtime_call(ac);
+
+ ac->init_req(ArrayCopyNode::Src, src);
+ ac->init_req(ArrayCopyNode::SrcPos, src_offset);
+ ac->init_req(ArrayCopyNode::Dest, dest);
+ ac->init_req(ArrayCopyNode::DestPos, dest_offset);
+ ac->init_req(ArrayCopyNode::Length, length);
+ ac->init_req(ArrayCopyNode::SrcLen, src_length);
+ ac->init_req(ArrayCopyNode::DestLen, dest_length);
+ ac->init_req(ArrayCopyNode::SrcKlass, src_klass);
+ ac->init_req(ArrayCopyNode::DestKlass, dest_klass);
+
+ if (may_throw) {
+ ac->set_req(TypeFunc::I_O , kit->i_o());
+ kit->add_safepoint_edges(ac, false);
+ }
+
+ return ac;
+}
+
+void ArrayCopyNode::connect_outputs(GraphKit* kit) {
+ kit->set_all_memory_call(this, true);
+ kit->set_control(kit->gvn().transform(new ProjNode(this,TypeFunc::Control)));
+ kit->set_i_o(kit->gvn().transform(new ProjNode(this, TypeFunc::I_O)));
+ kit->make_slow_call_ex(this, kit->env()->Throwable_klass(), true);
+ kit->set_all_memory_call(this);
+}
+
+#ifndef PRODUCT
+const char* ArrayCopyNode::_kind_names[] = {"arraycopy", "arraycopy, validated arguments", "clone", "oop array clone", "CopyOf", "CopyOfRange"};
+void ArrayCopyNode::dump_spec(outputStream *st) const {
+ CallNode::dump_spec(st);
+ st->print(" (%s%s)", _kind_names[_kind], _alloc_tightly_coupled ? ", tightly coupled allocation" : "");
+}
+#endif
--- a/hotspot/src/share/vm/opto/callnode.hpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/opto/callnode.hpp Thu Aug 21 19:06:21 2014 +0000
@@ -1063,4 +1063,108 @@
virtual bool guaranteed_safepoint() { return false; }
};
+class GraphKit;
+
+class ArrayCopyNode : public CallNode {
+private:
+
+ // What kind of arraycopy variant is this?
+ enum {
+ ArrayCopy, // System.arraycopy()
+ ArrayCopyNoTest, // System.arraycopy(), all arguments validated
+ CloneBasic, // A clone that can be copied by 64 bit chunks
+ CloneOop, // An oop array clone
+ CopyOf, // Arrays.copyOf()
+ CopyOfRange // Arrays.copyOfRange()
+ } _kind;
+
+#ifndef PRODUCT
+ static const char* _kind_names[CopyOfRange+1];
+#endif
+ // Is the alloc obtained with
+ // AllocateArrayNode::Ideal_array_allocation() tighly coupled
+ // (arraycopy follows immediately the allocation)?
+ // We cache the result of LibraryCallKit::tightly_coupled_allocation
+ // here because it's much easier to find whether there's a tightly
+ // couple allocation at parse time than at macro expansion time. At
+ // macro expansion time, for every use of the allocation node we
+ // would need to figure out whether it happens after the arraycopy (and
+ // can be ignored) or between the allocation and the arraycopy. At
+ // parse time, it's straightforward because whatever happens after
+ // the arraycopy is not parsed yet so doesn't exist when
+ // LibraryCallKit::tightly_coupled_allocation() is called.
+ bool _alloc_tightly_coupled;
+
+ static const TypeFunc* arraycopy_type() {
+ const Type** fields = TypeTuple::fields(ParmLimit - TypeFunc::Parms);
+ fields[Src] = TypeInstPtr::BOTTOM;
+ fields[SrcPos] = TypeInt::INT;
+ fields[Dest] = TypeInstPtr::BOTTOM;
+ fields[DestPos] = TypeInt::INT;
+ fields[Length] = TypeInt::INT;
+ fields[SrcLen] = TypeInt::INT;
+ fields[DestLen] = TypeInt::INT;
+ fields[SrcKlass] = TypeKlassPtr::BOTTOM;
+ fields[DestKlass] = TypeKlassPtr::BOTTOM;
+ const TypeTuple *domain = TypeTuple::make(ParmLimit, fields);
+
+ // create result type (range)
+ fields = TypeTuple::fields(0);
+
+ const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
+
+ return TypeFunc::make(domain, range);
+ }
+
+ ArrayCopyNode(Compile* C, bool alloc_tightly_coupled);
+
+public:
+
+ enum {
+ Src = TypeFunc::Parms,
+ SrcPos,
+ Dest,
+ DestPos,
+ Length,
+ SrcLen,
+ DestLen,
+ SrcKlass,
+ DestKlass,
+ ParmLimit
+ };
+
+ static ArrayCopyNode* make(GraphKit* kit, bool may_throw,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* length,
+ bool alloc_tightly_coupled,
+ Node* src_length = NULL, Node* dest_length = NULL,
+ Node* src_klass = NULL, Node* dest_klass = NULL);
+
+ void connect_outputs(GraphKit* kit);
+
+ bool is_arraycopy() const { return _kind == ArrayCopy; }
+ bool is_arraycopy_notest() const { return _kind == ArrayCopyNoTest; }
+ bool is_clonebasic() const { return _kind == CloneBasic; }
+ bool is_cloneoop() const { return _kind == CloneOop; }
+ bool is_copyof() const { return _kind == CopyOf; }
+ bool is_copyofrange() const { return _kind == CopyOfRange; }
+
+ void set_arraycopy() { _kind = ArrayCopy; }
+ void set_arraycopy_notest() { _kind = ArrayCopyNoTest; }
+ void set_clonebasic() { _kind = CloneBasic; }
+ void set_cloneoop() { _kind = CloneOop; }
+ void set_copyof() { _kind = CopyOf; }
+ void set_copyofrange() { _kind = CopyOfRange; }
+
+ virtual int Opcode() const;
+ virtual uint size_of() const; // Size is bigger
+ virtual bool guaranteed_safepoint() { return false; }
+
+ bool is_alloc_tightly_coupled() const { return _alloc_tightly_coupled; }
+
+#ifndef PRODUCT
+ virtual void dump_spec(outputStream *st) const;
+#endif
+};
#endif // SHARE_VM_OPTO_CALLNODE_HPP
--- a/hotspot/src/share/vm/opto/classes.hpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/opto/classes.hpp Thu Aug 21 19:06:21 2014 +0000
@@ -37,6 +37,7 @@
macro(AllocateArray)
macro(AndI)
macro(AndL)
+macro(ArrayCopy)
macro(AryEq)
macro(AtanD)
macro(Binary)
--- a/hotspot/src/share/vm/opto/compile.cpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/opto/compile.cpp Thu Aug 21 19:06:21 2014 +0000
@@ -3795,6 +3795,56 @@
}
}
+//----------------------------static_subtype_check-----------------------------
+// Shortcut important common cases when superklass is exact:
+// (0) superklass is java.lang.Object (can occur in reflective code)
+// (1) subklass is already limited to a subtype of superklass => always ok
+// (2) subklass does not overlap with superklass => always fail
+// (3) superklass has NO subtypes and we can check with a simple compare.
+int Compile::static_subtype_check(ciKlass* superk, ciKlass* subk) {
+ if (StressReflectiveCode) {
+ return SSC_full_test; // Let caller generate the general case.
+ }
+
+ if (superk == env()->Object_klass()) {
+ return SSC_always_true; // (0) this test cannot fail
+ }
+
+ ciType* superelem = superk;
+ if (superelem->is_array_klass())
+ superelem = superelem->as_array_klass()->base_element_type();
+
+ if (!subk->is_interface()) { // cannot trust static interface types yet
+ if (subk->is_subtype_of(superk)) {
+ return SSC_always_true; // (1) false path dead; no dynamic test needed
+ }
+ if (!(superelem->is_klass() && superelem->as_klass()->is_interface()) &&
+ !superk->is_subtype_of(subk)) {
+ return SSC_always_false;
+ }
+ }
+
+ // If casting to an instance klass, it must have no subtypes
+ if (superk->is_interface()) {
+ // Cannot trust interfaces yet.
+ // %%% S.B. superk->nof_implementors() == 1
+ } else if (superelem->is_instance_klass()) {
+ ciInstanceKlass* ik = superelem->as_instance_klass();
+ if (!ik->has_subklass() && !ik->is_interface()) {
+ if (!ik->is_final()) {
+ // Add a dependency if there is a chance of a later subclass.
+ dependencies()->assert_leaf_type(ik);
+ }
+ return SSC_easy_test; // (3) caller can do a simple ptr comparison
+ }
+ } else {
+ // A primitive array type has no subtypes.
+ return SSC_easy_test; // (3) caller can do a simple ptr comparison
+ }
+
+ return SSC_full_test;
+}
+
// The message about the current inlining is accumulated in
// _print_inlining_stream and transfered into the _print_inlining_list
// once we know whether inlining succeeds or not. For regular
--- a/hotspot/src/share/vm/opto/compile.hpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/opto/compile.hpp Thu Aug 21 19:06:21 2014 +0000
@@ -1200,6 +1200,10 @@
// Definitions of pd methods
static void pd_compiler2_init();
+ // Static parse-time type checking logic for gen_subtype_check:
+ enum { SSC_always_false, SSC_always_true, SSC_easy_test, SSC_full_test };
+ int static_subtype_check(ciKlass* superk, ciKlass* subk);
+
// Auxiliary method for randomized fuzzing/stressing
static bool randomized_select(int count);
};
--- a/hotspot/src/share/vm/opto/graphKit.cpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/opto/graphKit.cpp Thu Aug 21 19:06:21 2014 +0000
@@ -2520,6 +2520,21 @@
set_control(norm);
}
+static IfNode* gen_subtype_check_compare(Node* ctrl, Node* in1, Node* in2, BoolTest::mask test, float p, PhaseGVN* gvn, BasicType bt) {
+ Node* cmp = NULL;
+ switch(bt) {
+ case T_INT: cmp = new CmpINode(in1, in2); break;
+ case T_ADDRESS: cmp = new CmpPNode(in1, in2); break;
+ default: fatal(err_msg("unexpected comparison type %s", type2name(bt)));
+ }
+ gvn->transform(cmp);
+ Node* bol = gvn->transform(new BoolNode(cmp, test));
+ IfNode* iff = new IfNode(ctrl, bol, p, COUNT_UNKNOWN);
+ gvn->transform(iff);
+ if (!bol->is_Con()) gvn->record_for_igvn(iff);
+ return iff;
+}
+
//-------------------------------gen_subtype_check-----------------------------
// Generate a subtyping check. Takes as input the subtype and supertype.
@@ -2529,16 +2544,17 @@
// but that's not exposed to the optimizer. This call also doesn't take in an
// Object; if you wish to check an Object you need to load the Object's class
// prior to coming here.
-Node* GraphKit::gen_subtype_check(Node* subklass, Node* superklass) {
+Node* Phase::gen_subtype_check(Node* subklass, Node* superklass, Node** ctrl, MergeMemNode* mem, PhaseGVN* gvn) {
+ Compile* C = gvn->C;
// Fast check for identical types, perhaps identical constants.
// The types can even be identical non-constants, in cases
// involving Array.newInstance, Object.clone, etc.
if (subklass == superklass)
- return top(); // false path is dead; no test needed.
-
- if (_gvn.type(superklass)->singleton()) {
- ciKlass* superk = _gvn.type(superklass)->is_klassptr()->klass();
- ciKlass* subk = _gvn.type(subklass)->is_klassptr()->klass();
+ return C->top(); // false path is dead; no test needed.
+
+ if (gvn->type(superklass)->singleton()) {
+ ciKlass* superk = gvn->type(superklass)->is_klassptr()->klass();
+ ciKlass* subk = gvn->type(subklass)->is_klassptr()->klass();
// In the common case of an exact superklass, try to fold up the
// test before generating code. You may ask, why not just generate
@@ -2549,25 +2565,23 @@
// Foo[] fa = blah(); Foo x = fa[0]; fa[1] = x;
// Here, the type of 'fa' is often exact, so the store check
// of fa[1]=x will fold up, without testing the nullness of x.
- switch (static_subtype_check(superk, subk)) {
- case SSC_always_false:
+ switch (C->static_subtype_check(superk, subk)) {
+ case Compile::SSC_always_false:
{
- Node* always_fail = control();
- set_control(top());
+ Node* always_fail = *ctrl;
+ *ctrl = gvn->C->top();
return always_fail;
}
- case SSC_always_true:
- return top();
- case SSC_easy_test:
+ case Compile::SSC_always_true:
+ return C->top();
+ case Compile::SSC_easy_test:
{
// Just do a direct pointer compare and be done.
- Node* cmp = _gvn.transform( new CmpPNode(subklass, superklass) );
- Node* bol = _gvn.transform( new BoolNode(cmp, BoolTest::eq) );
- IfNode* iff = create_and_xform_if(control(), bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
- set_control( _gvn.transform( new IfTrueNode (iff) ) );
- return _gvn.transform( new IfFalseNode(iff) );
+ IfNode* iff = gen_subtype_check_compare(*ctrl, subklass, superklass, BoolTest::eq, PROB_STATIC_FREQUENT, gvn, T_ADDRESS);
+ *ctrl = gvn->transform(new IfTrueNode(iff));
+ return gvn->transform(new IfFalseNode(iff));
}
- case SSC_full_test:
+ case Compile::SSC_full_test:
break;
default:
ShouldNotReachHere();
@@ -2579,11 +2593,11 @@
// will always succeed. We could leave a dependency behind to ensure this.
// First load the super-klass's check-offset
- Node *p1 = basic_plus_adr( superklass, superklass, in_bytes(Klass::super_check_offset_offset()) );
- Node *chk_off = _gvn.transform(new LoadINode(NULL, memory(p1), p1, _gvn.type(p1)->is_ptr(),
- TypeInt::INT, MemNode::unordered));
+ Node *p1 = gvn->transform(new AddPNode(superklass, superklass, gvn->MakeConX(in_bytes(Klass::super_check_offset_offset()))));
+ Node* m = mem->memory_at(C->get_alias_index(gvn->type(p1)->is_ptr()));
+ Node *chk_off = gvn->transform(new LoadINode(NULL, m, p1, gvn->type(p1)->is_ptr(), TypeInt::INT, MemNode::unordered));
int cacheoff_con = in_bytes(Klass::secondary_super_cache_offset());
- bool might_be_cache = (find_int_con(chk_off, cacheoff_con) == cacheoff_con);
+ bool might_be_cache = (gvn->find_int_con(chk_off, cacheoff_con) == cacheoff_con);
// Load from the sub-klass's super-class display list, or a 1-word cache of
// the secondary superclass list, or a failing value with a sentinel offset
@@ -2591,42 +2605,44 @@
// hierarchy and we have to scan the secondary superclass list the hard way.
// Worst-case type is a little odd: NULL is allowed as a result (usually
// klass loads can never produce a NULL).
- Node *chk_off_X = ConvI2X(chk_off);
- Node *p2 = _gvn.transform( new AddPNode(subklass,subklass,chk_off_X) );
+ Node *chk_off_X = chk_off;
+#ifdef _LP64
+ chk_off_X = gvn->transform(new ConvI2LNode(chk_off_X));
+#endif
+ Node *p2 = gvn->transform(new AddPNode(subklass,subklass,chk_off_X));
// For some types like interfaces the following loadKlass is from a 1-word
// cache which is mutable so can't use immutable memory. Other
// types load from the super-class display table which is immutable.
- Node *kmem = might_be_cache ? memory(p2) : immutable_memory();
- Node *nkls = _gvn.transform( LoadKlassNode::make( _gvn, kmem, p2, _gvn.type(p2)->is_ptr(), TypeKlassPtr::OBJECT_OR_NULL ) );
+ m = mem->memory_at(C->get_alias_index(gvn->type(p2)->is_ptr()));
+ Node *kmem = might_be_cache ? m : C->immutable_memory();
+ Node *nkls = gvn->transform(LoadKlassNode::make(*gvn, kmem, p2, gvn->type(p2)->is_ptr(), TypeKlassPtr::OBJECT_OR_NULL));
// Compile speed common case: ARE a subtype and we canNOT fail
if( superklass == nkls )
- return top(); // false path is dead; no test needed.
+ return C->top(); // false path is dead; no test needed.
// See if we get an immediate positive hit. Happens roughly 83% of the
// time. Test to see if the value loaded just previously from the subklass
// is exactly the superklass.
- Node *cmp1 = _gvn.transform( new CmpPNode( superklass, nkls ) );
- Node *bol1 = _gvn.transform( new BoolNode( cmp1, BoolTest::eq ) );
- IfNode *iff1 = create_and_xform_if( control(), bol1, PROB_LIKELY(0.83f), COUNT_UNKNOWN );
- Node *iftrue1 = _gvn.transform( new IfTrueNode ( iff1 ) );
- set_control( _gvn.transform( new IfFalseNode( iff1 ) ) );
+ IfNode *iff1 = gen_subtype_check_compare(*ctrl, superklass, nkls, BoolTest::eq, PROB_LIKELY(0.83f), gvn, T_ADDRESS);
+ Node *iftrue1 = gvn->transform( new IfTrueNode (iff1));
+ *ctrl = gvn->transform(new IfFalseNode(iff1));
// Compile speed common case: Check for being deterministic right now. If
// chk_off is a constant and not equal to cacheoff then we are NOT a
// subklass. In this case we need exactly the 1 test above and we can
// return those results immediately.
if (!might_be_cache) {
- Node* not_subtype_ctrl = control();
- set_control(iftrue1); // We need exactly the 1 test above
+ Node* not_subtype_ctrl = *ctrl;
+ *ctrl = iftrue1; // We need exactly the 1 test above
return not_subtype_ctrl;
}
// Gather the various success & failures here
RegionNode *r_ok_subtype = new RegionNode(4);
- record_for_igvn(r_ok_subtype);
+ gvn->record_for_igvn(r_ok_subtype);
RegionNode *r_not_subtype = new RegionNode(3);
- record_for_igvn(r_not_subtype);
+ gvn->record_for_igvn(r_not_subtype);
r_ok_subtype->init_req(1, iftrue1);
@@ -2635,21 +2651,17 @@
// check-offset points into the subklass display list or the 1-element
// cache. If it points to the display (and NOT the cache) and the display
// missed then it's not a subtype.
- Node *cacheoff = _gvn.intcon(cacheoff_con);
- Node *cmp2 = _gvn.transform( new CmpINode( chk_off, cacheoff ) );
- Node *bol2 = _gvn.transform( new BoolNode( cmp2, BoolTest::ne ) );
- IfNode *iff2 = create_and_xform_if( control(), bol2, PROB_LIKELY(0.63f), COUNT_UNKNOWN );
- r_not_subtype->init_req(1, _gvn.transform( new IfTrueNode (iff2) ) );
- set_control( _gvn.transform( new IfFalseNode(iff2) ) );
+ Node *cacheoff = gvn->intcon(cacheoff_con);
+ IfNode *iff2 = gen_subtype_check_compare(*ctrl, chk_off, cacheoff, BoolTest::ne, PROB_LIKELY(0.63f), gvn, T_INT);
+ r_not_subtype->init_req(1, gvn->transform(new IfTrueNode (iff2)));
+ *ctrl = gvn->transform(new IfFalseNode(iff2));
// Check for self. Very rare to get here, but it is taken 1/3 the time.
// No performance impact (too rare) but allows sharing of secondary arrays
// which has some footprint reduction.
- Node *cmp3 = _gvn.transform( new CmpPNode( subklass, superklass ) );
- Node *bol3 = _gvn.transform( new BoolNode( cmp3, BoolTest::eq ) );
- IfNode *iff3 = create_and_xform_if( control(), bol3, PROB_LIKELY(0.36f), COUNT_UNKNOWN );
- r_ok_subtype->init_req(2, _gvn.transform( new IfTrueNode ( iff3 ) ) );
- set_control( _gvn.transform( new IfFalseNode( iff3 ) ) );
+ IfNode *iff3 = gen_subtype_check_compare(*ctrl, subklass, superklass, BoolTest::eq, PROB_LIKELY(0.36f), gvn, T_ADDRESS);
+ r_ok_subtype->init_req(2, gvn->transform(new IfTrueNode(iff3)));
+ *ctrl = gvn->transform(new IfFalseNode(iff3));
// -- Roads not taken here: --
// We could also have chosen to perform the self-check at the beginning
@@ -2672,68 +2684,16 @@
// out of line, and it can only improve I-cache density.
// The decision to inline or out-of-line this final check is platform
// dependent, and is found in the AD file definition of PartialSubtypeCheck.
- Node* psc = _gvn.transform(
- new PartialSubtypeCheckNode(control(), subklass, superklass) );
-
- Node *cmp4 = _gvn.transform( new CmpPNode( psc, null() ) );
- Node *bol4 = _gvn.transform( new BoolNode( cmp4, BoolTest::ne ) );
- IfNode *iff4 = create_and_xform_if( control(), bol4, PROB_FAIR, COUNT_UNKNOWN );
- r_not_subtype->init_req(2, _gvn.transform( new IfTrueNode (iff4) ) );
- r_ok_subtype ->init_req(3, _gvn.transform( new IfFalseNode(iff4) ) );
+ Node* psc = gvn->transform(
+ new PartialSubtypeCheckNode(*ctrl, subklass, superklass));
+
+ IfNode *iff4 = gen_subtype_check_compare(*ctrl, psc, gvn->zerocon(T_OBJECT), BoolTest::ne, PROB_FAIR, gvn, T_ADDRESS);
+ r_not_subtype->init_req(2, gvn->transform(new IfTrueNode (iff4)));
+ r_ok_subtype ->init_req(3, gvn->transform(new IfFalseNode(iff4)));
// Return false path; set default control to true path.
- set_control( _gvn.transform(r_ok_subtype) );
- return _gvn.transform(r_not_subtype);
-}
-
-//----------------------------static_subtype_check-----------------------------
-// Shortcut important common cases when superklass is exact:
-// (0) superklass is java.lang.Object (can occur in reflective code)
-// (1) subklass is already limited to a subtype of superklass => always ok
-// (2) subklass does not overlap with superklass => always fail
-// (3) superklass has NO subtypes and we can check with a simple compare.
-int GraphKit::static_subtype_check(ciKlass* superk, ciKlass* subk) {
- if (StressReflectiveCode) {
- return SSC_full_test; // Let caller generate the general case.
- }
-
- if (superk == env()->Object_klass()) {
- return SSC_always_true; // (0) this test cannot fail
- }
-
- ciType* superelem = superk;
- if (superelem->is_array_klass())
- superelem = superelem->as_array_klass()->base_element_type();
-
- if (!subk->is_interface()) { // cannot trust static interface types yet
- if (subk->is_subtype_of(superk)) {
- return SSC_always_true; // (1) false path dead; no dynamic test needed
- }
- if (!(superelem->is_klass() && superelem->as_klass()->is_interface()) &&
- !superk->is_subtype_of(subk)) {
- return SSC_always_false;
- }
- }
-
- // If casting to an instance klass, it must have no subtypes
- if (superk->is_interface()) {
- // Cannot trust interfaces yet.
- // %%% S.B. superk->nof_implementors() == 1
- } else if (superelem->is_instance_klass()) {
- ciInstanceKlass* ik = superelem->as_instance_klass();
- if (!ik->has_subklass() && !ik->is_interface()) {
- if (!ik->is_final()) {
- // Add a dependency if there is a chance of a later subclass.
- C->dependencies()->assert_leaf_type(ik);
- }
- return SSC_easy_test; // (3) caller can do a simple ptr comparison
- }
- } else {
- // A primitive array type has no subtypes.
- return SSC_easy_test; // (3) caller can do a simple ptr comparison
- }
-
- return SSC_full_test;
+ *ctrl = gvn->transform(r_ok_subtype);
+ return gvn->transform(r_not_subtype);
}
// Profile-driven exact type check:
@@ -2813,7 +2773,7 @@
ciKlass* exact_kls = spec_klass == NULL ? profile_has_unique_klass() : spec_klass;
if (exact_kls != NULL) {// no cast failures here
if (require_klass == NULL ||
- static_subtype_check(require_klass, exact_kls) == SSC_always_true) {
+ C->static_subtype_check(require_klass, exact_kls) == Compile::SSC_always_true) {
// If we narrow the type to match what the type profile sees or
// the speculative type, we can then remove the rest of the
// cast.
@@ -2833,7 +2793,7 @@
}
return exact_obj;
}
- // assert(ssc == SSC_always_true)... except maybe the profile lied to us.
+ // assert(ssc == Compile::SSC_always_true)... except maybe the profile lied to us.
}
return NULL;
@@ -2938,8 +2898,8 @@
ciKlass* superk = _gvn.type(superklass)->is_klassptr()->klass();
ciKlass* subk = _gvn.type(obj)->is_oopptr()->klass();
if (subk != NULL && subk->is_loaded()) {
- int static_res = static_subtype_check(superk, subk);
- known_statically = (static_res == SSC_always_true || static_res == SSC_always_false);
+ int static_res = C->static_subtype_check(superk, subk);
+ known_statically = (static_res == Compile::SSC_always_true || static_res == Compile::SSC_always_false);
}
}
@@ -3007,13 +2967,13 @@
if (tk->singleton()) {
const TypeOopPtr* objtp = _gvn.type(obj)->isa_oopptr();
if (objtp != NULL && objtp->klass() != NULL) {
- switch (static_subtype_check(tk->klass(), objtp->klass())) {
- case SSC_always_true:
+ switch (C->static_subtype_check(tk->klass(), objtp->klass())) {
+ case Compile::SSC_always_true:
// If we know the type check always succeed then we don't use
// the profiling data at this bytecode. Don't lose it, feed it
// to the type system as a speculative type.
return record_profiled_receiver_for_speculation(obj);
- case SSC_always_false:
+ case Compile::SSC_always_false:
// It needs a null check because a null will *pass* the cast check.
// A non-null value will always produce an exception.
return null_assert(obj);
--- a/hotspot/src/share/vm/opto/graphKit.hpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/opto/graphKit.hpp Thu Aug 21 19:06:21 2014 +0000
@@ -829,17 +829,13 @@
Node* gen_checkcast( Node *subobj, Node* superkls,
Node* *failure_control = NULL );
- // Generate a subtyping check. Takes as input the subtype and supertype.
- // Returns 2 values: sets the default control() to the true path and
- // returns the false path. Only reads from constant memory taken from the
- // default memory; does not write anything. It also doesn't take in an
- // Object; if you wish to check an Object you need to load the Object's
- // class prior to coming here.
- Node* gen_subtype_check(Node* subklass, Node* superklass);
-
- // Static parse-time type checking logic for gen_subtype_check:
- enum { SSC_always_false, SSC_always_true, SSC_easy_test, SSC_full_test };
- int static_subtype_check(ciKlass* superk, ciKlass* subk);
+ Node* gen_subtype_check(Node* subklass, Node* superklass) {
+ MergeMemNode* mem = merged_memory();
+ Node* ctrl = control();
+ Node* n = Phase::gen_subtype_check(subklass, superklass, &ctrl, mem, &_gvn);
+ set_control(ctrl);
+ return n;
+ }
// Exact type check used for predicted calls and casts.
// Rewrites (*casted_receiver) to be casted to the stronger type.
--- a/hotspot/src/share/vm/opto/ifnode.cpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/opto/ifnode.cpp Thu Aug 21 19:06:21 2014 +0000
@@ -503,7 +503,7 @@
jint off = 0;
if (l->is_top()) {
return 0;
- } else if (l->is_Add()) {
+ } else if (l->Opcode() == Op_AddI) {
if ((off = l->in(1)->find_int_con(0)) != 0) {
ind = l->in(2);
} else if ((off = l->in(2)->find_int_con(0)) != 0) {
--- a/hotspot/src/share/vm/opto/library_call.cpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/opto/library_call.cpp Thu Aug 21 19:06:21 2014 +0000
@@ -146,15 +146,10 @@
Node* generate_negative_guard(Node* index, RegionNode* region,
// resulting CastII of index:
Node* *pos_index = NULL);
- Node* generate_nonpositive_guard(Node* index, bool never_negative,
- // resulting CastII of index:
- Node* *pos_index = NULL);
Node* generate_limit_guard(Node* offset, Node* subseq_length,
Node* array_length,
RegionNode* region);
Node* generate_current_thread(Node* &tls_output);
- address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,
- bool disjoint_bases, const char* &name, bool dest_uninitialized);
Node* load_mirror_from_klass(Node* klass);
Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
RegionNode* region, int null_path,
@@ -264,47 +259,8 @@
// Helper functions for inlining arraycopy
bool inline_arraycopy();
- void generate_arraycopy(const TypePtr* adr_type,
- BasicType basic_elem_type,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length,
- bool disjoint_bases = false,
- bool length_never_negative = false,
- RegionNode* slow_region = NULL);
AllocateArrayNode* tightly_coupled_allocation(Node* ptr,
RegionNode* slow_region);
- void generate_clear_array(const TypePtr* adr_type,
- Node* dest,
- BasicType basic_elem_type,
- Node* slice_off,
- Node* slice_len,
- Node* slice_end);
- bool generate_block_arraycopy(const TypePtr* adr_type,
- BasicType basic_elem_type,
- AllocateNode* alloc,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* dest_size, bool dest_uninitialized);
- void generate_slow_arraycopy(const TypePtr* adr_type,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length, bool dest_uninitialized);
- Node* generate_checkcast_arraycopy(const TypePtr* adr_type,
- Node* dest_elem_klass,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length, bool dest_uninitialized);
- Node* generate_generic_arraycopy(const TypePtr* adr_type,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length, bool dest_uninitialized);
- void generate_unchecked_arraycopy(const TypePtr* adr_type,
- BasicType basic_elem_type,
- bool disjoint_bases,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length, bool dest_uninitialized);
typedef enum { LS_xadd, LS_xchg, LS_cmpxchg } LoadStoreKind;
bool inline_unsafe_load_store(BasicType type, LoadStoreKind kind);
bool inline_unsafe_ordered_store(BasicType type);
@@ -1049,25 +1005,6 @@
return is_neg;
}
-inline Node* LibraryCallKit::generate_nonpositive_guard(Node* index, bool never_negative,
- Node* *pos_index) {
- if (stopped())
- return NULL; // already stopped
- if (_gvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
- return NULL; // index is already adequately typed
- Node* cmp_le = _gvn.transform(new CmpINode(index, intcon(0)));
- BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
- Node* bol_le = _gvn.transform(new BoolNode(cmp_le, le_or_eq));
- Node* is_notp = generate_guard(bol_le, NULL, PROB_MIN);
- if (is_notp != NULL && pos_index != NULL) {
- // Emulate effect of Parse::adjust_map_after_if.
- Node* ccast = new CastIINode(index, TypeInt::POS1);
- ccast->set_req(0, control());
- (*pos_index) = _gvn.transform(ccast);
- }
- return is_notp;
-}
-
// Make sure that 'position' is a valid limit index, in [0..length].
// There are two equivalent plans for checking this:
// A. (offset + copyLength) unsigned<= arrayLength
@@ -2658,7 +2595,8 @@
if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
if (!is_store) {
- Node* p = make_load(control(), adr, value_type, type, adr_type, MemNode::unordered, is_volatile);
+ MemNode::MemOrd mo = is_volatile ? MemNode::acquire : MemNode::unordered;
+ Node* p = make_load(control(), adr, value_type, type, adr_type, mo, is_volatile);
// load value
switch (type) {
case T_BOOLEAN:
@@ -3928,13 +3866,18 @@
// oop stores need checking.
// Extreme case: Arrays.copyOf((Integer[])x, 10, String[].class).
// This will fail a store-check if x contains any non-nulls.
- bool disjoint_bases = true;
- // if start > orig_length then the length of the copy may be
- // negative.
- bool length_never_negative = !is_copyOfRange;
- generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
- original, start, newcopy, intcon(0), moved,
- disjoint_bases, length_never_negative);
+
+ Node* alloc = tightly_coupled_allocation(newcopy, NULL);
+
+ ArrayCopyNode* ac = ArrayCopyNode::make(this, true, original, start, newcopy, intcon(0), moved, alloc != NULL);
+ if (!is_copyOfRange) {
+ ac->set_copyof();
+ } else {
+ ac->set_copyofrange();
+ }
+ Node* n = _gvn.transform(ac);
+ assert(n == ac, "cannot disappear");
+ ac->connect_outputs(this);
}
} // original reexecute is set back here
@@ -4445,10 +4388,12 @@
countx = _gvn.transform(new URShiftXNode(countx, intcon(LogBytesPerLong) ));
const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
- bool disjoint_bases = true;
- generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
- src, NULL, dest, NULL, countx,
- /*dest_uninitialized*/true);
+
+ ArrayCopyNode* ac = ArrayCopyNode::make(this, false, src, NULL, dest, NULL, countx, false);
+ ac->set_clonebasic();
+ Node* n = _gvn.transform(ac);
+ assert(n == ac, "cannot disappear");
+ set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type);
// If necessary, emit some card marks afterwards. (Non-arrays only.)
if (card_mark) {
@@ -4557,12 +4502,13 @@
PreserveJVMState pjvms2(this);
set_control(is_obja);
// Generate a direct call to the right arraycopy function(s).
- bool disjoint_bases = true;
- bool length_never_negative = true;
- generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
- obj, intcon(0), alloc_obj, intcon(0),
- obj_length,
- disjoint_bases, length_never_negative);
+ Node* alloc = tightly_coupled_allocation(alloc_obj, NULL);
+ ArrayCopyNode* ac = ArrayCopyNode::make(this, true, obj, intcon(0), alloc_obj, intcon(0), obj_length, alloc != NULL);
+ ac->set_cloneoop();
+ Node* n = _gvn.transform(ac);
+ assert(n == ac, "cannot disappear");
+ ac->connect_outputs(this);
+
result_reg->init_req(_objArray_path, control());
result_val->init_req(_objArray_path, alloc_obj);
result_i_o ->set_req(_objArray_path, i_o());
@@ -4656,42 +4602,6 @@
return true;
}
-//------------------------------basictype2arraycopy----------------------------
-address LibraryCallKit::basictype2arraycopy(BasicType t,
- Node* src_offset,
- Node* dest_offset,
- bool disjoint_bases,
- const char* &name,
- bool dest_uninitialized) {
- const TypeInt* src_offset_inttype = gvn().find_int_type(src_offset);;
- const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
-
- bool aligned = false;
- bool disjoint = disjoint_bases;
-
- // if the offsets are the same, we can treat the memory regions as
- // disjoint, because either the memory regions are in different arrays,
- // or they are identical (which we can treat as disjoint.) We can also
- // treat a copy with a destination index less that the source index
- // as disjoint since a low->high copy will work correctly in this case.
- if (src_offset_inttype != NULL && src_offset_inttype->is_con() &&
- dest_offset_inttype != NULL && dest_offset_inttype->is_con()) {
- // both indices are constants
- int s_offs = src_offset_inttype->get_con();
- int d_offs = dest_offset_inttype->get_con();
- int element_size = type2aelembytes(t);
- aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) &&
- ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0);
- if (s_offs >= d_offs) disjoint = true;
- } else if (src_offset == dest_offset && src_offset != NULL) {
- // This can occur if the offsets are identical non-constants.
- disjoint = true;
- }
-
- return StubRoutines::select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized);
-}
-
-
//------------------------------inline_arraycopy-----------------------
// public static native void java.lang.System.arraycopy(Object src, int srcPos,
// Object dest, int destPos,
@@ -4704,13 +4614,26 @@
Node* dest_offset = argument(3); // type: int
Node* length = argument(4); // type: int
- // Compile time checks. If any of these checks cannot be verified at compile time,
- // we do not make a fast path for this call. Instead, we let the call remain as it
- // is. The checks we choose to mandate at compile time are:
- //
+ // The following tests must be performed
// (1) src and dest are arrays.
- const Type* src_type = src->Value(&_gvn);
- const Type* dest_type = dest->Value(&_gvn);
+ // (2) src and dest arrays must have elements of the same BasicType
+ // (3) src and dest must not be null.
+ // (4) src_offset must not be negative.
+ // (5) dest_offset must not be negative.
+ // (6) length must not be negative.
+ // (7) src_offset + length must not exceed length of src.
+ // (8) dest_offset + length must not exceed length of dest.
+ // (9) each element of an oop array must be assignable
+
+ // (3) src and dest must not be null.
+ // always do this here because we need the JVM state for uncommon traps
+ src = null_check(src, T_ARRAY);
+ dest = null_check(dest, T_ARRAY);
+
+ bool notest = false;
+
+ const Type* src_type = _gvn.type(src);
+ const Type* dest_type = _gvn.type(dest);
const TypeAryPtr* top_src = src_type->isa_aryptr();
const TypeAryPtr* top_dest = dest_type->isa_aryptr();
@@ -4768,556 +4691,119 @@
}
}
- if (!has_src || !has_dest) {
- // Conservatively insert a memory barrier on all memory slices.
- // Do not let writes into the source float below the arraycopy.
- insert_mem_bar(Op_MemBarCPUOrder);
-
- // Call StubRoutines::generic_arraycopy stub.
- generate_arraycopy(TypeRawPtr::BOTTOM, T_CONFLICT,
- src, src_offset, dest, dest_offset, length);
-
- // Do not let reads from the destination float above the arraycopy.
- // Since we cannot type the arrays, we don't know which slices
- // might be affected. We could restrict this barrier only to those
- // memory slices which pertain to array elements--but don't bother.
- if (!InsertMemBarAfterArraycopy)
- // (If InsertMemBarAfterArraycopy, there is already one in place.)
- insert_mem_bar(Op_MemBarCPUOrder);
- return true;
- }
-
- // (2) src and dest arrays must have elements of the same BasicType
- // Figure out the size and type of the elements we will be copying.
- BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type();
- BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
- if (src_elem == T_ARRAY) src_elem = T_OBJECT;
- if (dest_elem == T_ARRAY) dest_elem = T_OBJECT;
-
- if (src_elem != dest_elem || dest_elem == T_VOID) {
- // The component types are not the same or are not recognized. Punt.
- // (But, avoid the native method wrapper to JVM_ArrayCopy.)
- generate_slow_arraycopy(TypePtr::BOTTOM,
- src, src_offset, dest, dest_offset, length,
- /*dest_uninitialized*/false);
- return true;
- }
-
- if (src_elem == T_OBJECT) {
- // If both arrays are object arrays then having the exact types
- // for both will remove the need for a subtype check at runtime
- // before the call and may make it possible to pick a faster copy
- // routine (without a subtype check on every element)
- // Do we have the exact type of src?
- bool could_have_src = src_spec;
- // Do we have the exact type of dest?
- bool could_have_dest = dest_spec;
- ciKlass* src_k = top_src->klass();
- ciKlass* dest_k = top_dest->klass();
- if (!src_spec) {
- src_k = src_type->speculative_type_not_null();
- if (src_k != NULL && src_k->is_array_klass()) {
+ if (has_src && has_dest) {
+ BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type();
+ BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
+ if (src_elem == T_ARRAY) src_elem = T_OBJECT;
+ if (dest_elem == T_ARRAY) dest_elem = T_OBJECT;
+
+ if (src_elem == dest_elem && src_elem == T_OBJECT) {
+ // If both arrays are object arrays then having the exact types
+ // for both will remove the need for a subtype check at runtime
+ // before the call and may make it possible to pick a faster copy
+ // routine (without a subtype check on every element)
+ // Do we have the exact type of src?
+ bool could_have_src = src_spec;
+ // Do we have the exact type of dest?
+ bool could_have_dest = dest_spec;
+ ciKlass* src_k = top_src->klass();
+ ciKlass* dest_k = top_dest->klass();
+ if (!src_spec) {
+ src_k = src_type->speculative_type_not_null();
+ if (src_k != NULL && src_k->is_array_klass()) {
could_have_src = true;
+ }
}
- }
- if (!dest_spec) {
- dest_k = dest_type->speculative_type_not_null();
- if (dest_k != NULL && dest_k->is_array_klass()) {
- could_have_dest = true;
+ if (!dest_spec) {
+ dest_k = dest_type->speculative_type_not_null();
+ if (dest_k != NULL && dest_k->is_array_klass()) {
+ could_have_dest = true;
+ }
}
- }
- if (could_have_src && could_have_dest) {
- // If we can have both exact types, emit the missing guards
- if (could_have_src && !src_spec) {
- src = maybe_cast_profiled_obj(src, src_k);
- }
- if (could_have_dest && !dest_spec) {
- dest = maybe_cast_profiled_obj(dest, dest_k);
+ if (could_have_src && could_have_dest) {
+ // If we can have both exact types, emit the missing guards
+ if (could_have_src && !src_spec) {
+ src = maybe_cast_profiled_obj(src, src_k);
+ }
+ if (could_have_dest && !dest_spec) {
+ dest = maybe_cast_profiled_obj(dest, dest_k);
+ }
}
}
}
- //---------------------------------------------------------------------------
- // We will make a fast path for this call to arraycopy.
-
- // We have the following tests left to perform:
- //
- // (3) src and dest must not be null.
- // (4) src_offset must not be negative.
- // (5) dest_offset must not be negative.
- // (6) length must not be negative.
- // (7) src_offset + length must not exceed length of src.
- // (8) dest_offset + length must not exceed length of dest.
- // (9) each element of an oop array must be assignable
-
- RegionNode* slow_region = new RegionNode(1);
- record_for_igvn(slow_region);
-
- // (3) operands must not be null
- // We currently perform our null checks with the null_check routine.
- // This means that the null exceptions will be reported in the caller
- // rather than (correctly) reported inside of the native arraycopy call.
- // This should be corrected, given time. We do our null check with the
- // stack pointer restored.
- src = null_check(src, T_ARRAY);
- dest = null_check(dest, T_ARRAY);
-
- // (4) src_offset must not be negative.
- generate_negative_guard(src_offset, slow_region);
-
- // (5) dest_offset must not be negative.
- generate_negative_guard(dest_offset, slow_region);
-
- // (6) length must not be negative (moved to generate_arraycopy()).
- // generate_negative_guard(length, slow_region);
-
- // (7) src_offset + length must not exceed length of src.
- generate_limit_guard(src_offset, length,
- load_array_length(src),
- slow_region);
-
- // (8) dest_offset + length must not exceed length of dest.
- generate_limit_guard(dest_offset, length,
- load_array_length(dest),
- slow_region);
-
- // (9) each element of an oop array must be assignable
- // The generate_arraycopy subroutine checks this.
-
- // This is where the memory effects are placed:
- const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem);
- generate_arraycopy(adr_type, dest_elem,
- src, src_offset, dest, dest_offset, length,
- false, false, slow_region);
-
- return true;
-}
-
-//-----------------------------generate_arraycopy----------------------
-// Generate an optimized call to arraycopy.
-// Caller must guard against non-arrays.
-// Caller must determine a common array basic-type for both arrays.
-// Caller must validate offsets against array bounds.
-// The slow_region has already collected guard failure paths
-// (such as out of bounds length or non-conformable array types).
-// The generated code has this shape, in general:
-//
-// if (length == 0) return // via zero_path
-// slowval = -1
-// if (types unknown) {
-// slowval = call generic copy loop
-// if (slowval == 0) return // via checked_path
-// } else if (indexes in bounds) {
-// if ((is object array) && !(array type check)) {
-// slowval = call checked copy loop
-// if (slowval == 0) return // via checked_path
-// } else {
-// call bulk copy loop
-// return // via fast_path
-// }
-// }
-// // adjust params for remaining work:
-// if (slowval != -1) {
-// n = -1^slowval; src_offset += n; dest_offset += n; length -= n
-// }
-// slow_region:
-// call slow arraycopy(src, src_offset, dest, dest_offset, length)
-// return // via slow_call_path
-//
-// This routine is used from several intrinsics: System.arraycopy,
-// Object.clone (the array subcase), and Arrays.copyOf[Range].
-//
-void
-LibraryCallKit::generate_arraycopy(const TypePtr* adr_type,
- BasicType basic_elem_type,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length,
- bool disjoint_bases,
- bool length_never_negative,
- RegionNode* slow_region) {
-
- if (slow_region == NULL) {
- slow_region = new RegionNode(1);
+ if (!too_many_traps(Deoptimization::Reason_intrinsic) && !src->is_top() && !dest->is_top()) {
+ // validate arguments: enables transformation the ArrayCopyNode
+ notest = true;
+
+ RegionNode* slow_region = new RegionNode(1);
record_for_igvn(slow_region);
- }
-
- Node* original_dest = dest;
- AllocateArrayNode* alloc = NULL; // used for zeroing, if needed
- bool dest_uninitialized = false;
-
- // See if this is the initialization of a newly-allocated array.
- // If so, we will take responsibility here for initializing it to zero.
- // (Note: Because tightly_coupled_allocation performs checks on the
- // out-edges of the dest, we need to avoid making derived pointers
- // from it until we have checked its uses.)
- if (ReduceBulkZeroing
- && !ZeroTLAB // pointless if already zeroed
- && basic_elem_type != T_CONFLICT // avoid corner case
- && !src->eqv_uncast(dest)
- && ((alloc = tightly_coupled_allocation(dest, slow_region))
- != NULL)
- && _gvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0
- && alloc->maybe_set_complete(&_gvn)) {
- // "You break it, you buy it."
- InitializeNode* init = alloc->initialization();
- assert(init->is_complete(), "we just did this");
- init->set_complete_with_arraycopy();
- assert(dest->is_CheckCastPP(), "sanity");
- assert(dest->in(0)->in(0) == init, "dest pinned");
- adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory
- // From this point on, every exit path is responsible for
- // initializing any non-copied parts of the object to zero.
- // Also, if this flag is set we make sure that arraycopy interacts properly
- // with G1, eliding pre-barriers. See CR 6627983.
- dest_uninitialized = true;
- } else {
- // No zeroing elimination here.
- alloc = NULL;
- //original_dest = dest;
- //dest_uninitialized = false;
- }
-
- // Results are placed here:
- enum { fast_path = 1, // normal void-returning assembly stub
- checked_path = 2, // special assembly stub with cleanup
- slow_call_path = 3, // something went wrong; call the VM
- zero_path = 4, // bypass when length of copy is zero
- bcopy_path = 5, // copy primitive array by 64-bit blocks
- PATH_LIMIT = 6
- };
- RegionNode* result_region = new RegionNode(PATH_LIMIT);
- PhiNode* result_i_o = new PhiNode(result_region, Type::ABIO);
- PhiNode* result_memory = new PhiNode(result_region, Type::MEMORY, adr_type);
- record_for_igvn(result_region);
- _gvn.set_type_bottom(result_i_o);
- _gvn.set_type_bottom(result_memory);
- assert(adr_type != TypePtr::BOTTOM, "must be RawMem or a T[] slice");
-
- // The slow_control path:
- Node* slow_control;
- Node* slow_i_o = i_o();
- Node* slow_mem = memory(adr_type);
- debug_only(slow_control = (Node*) badAddress);
-
- // Checked control path:
- Node* checked_control = top();
- Node* checked_mem = NULL;
- Node* checked_i_o = NULL;
- Node* checked_value = NULL;
-
- if (basic_elem_type == T_CONFLICT) {
- assert(!dest_uninitialized, "");
- Node* cv = generate_generic_arraycopy(adr_type,
- src, src_offset, dest, dest_offset,
- copy_length, dest_uninitialized);
- if (cv == NULL) cv = intcon(-1); // failure (no stub available)
- checked_control = control();
- checked_i_o = i_o();
- checked_mem = memory(adr_type);
- checked_value = cv;
- set_control(top()); // no fast path
- }
-
- Node* not_pos = generate_nonpositive_guard(copy_length, length_never_negative);
- if (not_pos != NULL) {
- PreserveJVMState pjvms(this);
- set_control(not_pos);
-
- // (6) length must not be negative.
- if (!length_never_negative) {
- generate_negative_guard(copy_length, slow_region);
- }
-
- // copy_length is 0.
- if (!stopped() && dest_uninitialized) {
- Node* dest_length = alloc->in(AllocateNode::ALength);
- if (copy_length->eqv_uncast(dest_length)
- || _gvn.find_int_con(dest_length, 1) <= 0) {
- // There is no zeroing to do. No need for a secondary raw memory barrier.
- } else {
- // Clear the whole thing since there are no source elements to copy.
- generate_clear_array(adr_type, dest, basic_elem_type,
- intcon(0), NULL,
- alloc->in(AllocateNode::AllocSize));
- // Use a secondary InitializeNode as raw memory barrier.
- // Currently it is needed only on this path since other
- // paths have stub or runtime calls as raw memory barriers.
- InitializeNode* init = insert_mem_bar_volatile(Op_Initialize,
- Compile::AliasIdxRaw,
- top())->as_Initialize();
- init->set_complete(&_gvn); // (there is no corresponding AllocateNode)
- }
- }
-
- // Present the results of the fast call.
- result_region->init_req(zero_path, control());
- result_i_o ->init_req(zero_path, i_o());
- result_memory->init_req(zero_path, memory(adr_type));
- }
-
- if (!stopped() && dest_uninitialized) {
- // We have to initialize the *uncopied* part of the array to zero.
- // The copy destination is the slice dest[off..off+len]. The other slices
- // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
- Node* dest_size = alloc->in(AllocateNode::AllocSize);
- Node* dest_length = alloc->in(AllocateNode::ALength);
- Node* dest_tail = _gvn.transform(new AddINode(dest_offset, copy_length));
-
- // If there is a head section that needs zeroing, do it now.
- if (find_int_con(dest_offset, -1) != 0) {
- generate_clear_array(adr_type, dest, basic_elem_type,
- intcon(0), dest_offset,
- NULL);
- }
-
- // Next, perform a dynamic check on the tail length.
- // It is often zero, and we can win big if we prove this.
- // There are two wins: Avoid generating the ClearArray
- // with its attendant messy index arithmetic, and upgrade
- // the copy to a more hardware-friendly word size of 64 bits.
- Node* tail_ctl = NULL;
- if (!stopped() && !dest_tail->eqv_uncast(dest_length)) {
- Node* cmp_lt = _gvn.transform(new CmpINode(dest_tail, dest_length));
- Node* bol_lt = _gvn.transform(new BoolNode(cmp_lt, BoolTest::lt));
- tail_ctl = generate_slow_guard(bol_lt, NULL);
- assert(tail_ctl != NULL || !stopped(), "must be an outcome");
- }
-
- // At this point, let's assume there is no tail.
- if (!stopped() && alloc != NULL && basic_elem_type != T_OBJECT) {
- // There is no tail. Try an upgrade to a 64-bit copy.
- bool didit = false;
- { PreserveJVMState pjvms(this);
- didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
- src, src_offset, dest, dest_offset,
- dest_size, dest_uninitialized);
- if (didit) {
- // Present the results of the block-copying fast call.
- result_region->init_req(bcopy_path, control());
- result_i_o ->init_req(bcopy_path, i_o());
- result_memory->init_req(bcopy_path, memory(adr_type));
- }
- }
- if (didit)
- set_control(top()); // no regular fast path
- }
-
- // Clear the tail, if any.
- if (tail_ctl != NULL) {
- Node* notail_ctl = stopped() ? NULL : control();
- set_control(tail_ctl);
- if (notail_ctl == NULL) {
- generate_clear_array(adr_type, dest, basic_elem_type,
- dest_tail, NULL,
- dest_size);
- } else {
- // Make a local merge.
- Node* done_ctl = new RegionNode(3);
- Node* done_mem = new PhiNode(done_ctl, Type::MEMORY, adr_type);
- done_ctl->init_req(1, notail_ctl);
- done_mem->init_req(1, memory(adr_type));
- generate_clear_array(adr_type, dest, basic_elem_type,
- dest_tail, NULL,
- dest_size);
- done_ctl->init_req(2, control());
- done_mem->init_req(2, memory(adr_type));
- set_control( _gvn.transform(done_ctl));
- set_memory( _gvn.transform(done_mem), adr_type );
- }
- }
- }
-
- BasicType copy_type = basic_elem_type;
- assert(basic_elem_type != T_ARRAY, "caller must fix this");
- if (!stopped() && copy_type == T_OBJECT) {
- // If src and dest have compatible element types, we can copy bits.
- // Types S[] and D[] are compatible if D is a supertype of S.
- //
- // If they are not, we will use checked_oop_disjoint_arraycopy,
- // which performs a fast optimistic per-oop check, and backs off
- // further to JVM_ArrayCopy on the first per-oop check that fails.
- // (Actually, we don't move raw bits only; the GC requires card marks.)
-
- // Get the Klass* for both src and dest
+
+ // (1) src and dest are arrays.
+ generate_non_array_guard(load_object_klass(src), slow_region);
+ generate_non_array_guard(load_object_klass(dest), slow_region);
+
+ // (2) src and dest arrays must have elements of the same BasicType
+ // done at macro expansion or at Ideal transformation time
+
+ // (4) src_offset must not be negative.
+ generate_negative_guard(src_offset, slow_region);
+
+ // (5) dest_offset must not be negative.
+ generate_negative_guard(dest_offset, slow_region);
+
+ // (7) src_offset + length must not exceed length of src.
+ generate_limit_guard(src_offset, length,
+ load_array_length(src),
+ slow_region);
+
+ // (8) dest_offset + length must not exceed length of dest.
+ generate_limit_guard(dest_offset, length,
+ load_array_length(dest),
+ slow_region);
+
+ // (9) each element of an oop array must be assignable
Node* src_klass = load_object_klass(src);
Node* dest_klass = load_object_klass(dest);
-
- // Generate the subtype check.
- // This might fold up statically, or then again it might not.
- //
- // Non-static example: Copying List<String>.elements to a new String[].
- // The backing store for a List<String> is always an Object[],
- // but its elements are always type String, if the generic types
- // are correct at the source level.
- //
- // Test S[] against D[], not S against D, because (probably)
- // the secondary supertype cache is less busy for S[] than S.
- // This usually only matters when D is an interface.
Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass);
- // Plug failing path into checked_oop_disjoint_arraycopy
+
if (not_subtype_ctrl != top()) {
PreserveJVMState pjvms(this);
set_control(not_subtype_ctrl);
- // (At this point we can assume disjoint_bases, since types differ.)
- int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
- Node* p1 = basic_plus_adr(dest_klass, ek_offset);
- Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM);
- Node* dest_elem_klass = _gvn.transform(n1);
- Node* cv = generate_checkcast_arraycopy(adr_type,
- dest_elem_klass,
- src, src_offset, dest, dest_offset,
- ConvI2X(copy_length), dest_uninitialized);
- if (cv == NULL) cv = intcon(-1); // failure (no stub available)
- checked_control = control();
- checked_i_o = i_o();
- checked_mem = memory(adr_type);
- checked_value = cv;
+ uncommon_trap(Deoptimization::Reason_intrinsic,
+ Deoptimization::Action_make_not_entrant);
+ assert(stopped(), "Should be stopped");
}
- // At this point we know we do not need type checks on oop stores.
-
- // Let's see if we need card marks:
- if (alloc != NULL && use_ReduceInitialCardMarks()) {
- // If we do not need card marks, copy using the jint or jlong stub.
- copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
- assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
- "sizes agree");
+ {
+ PreserveJVMState pjvms(this);
+ set_control(_gvn.transform(slow_region));
+ uncommon_trap(Deoptimization::Reason_intrinsic,
+ Deoptimization::Action_make_not_entrant);
+ assert(stopped(), "Should be stopped");
}
}
- if (!stopped()) {
- // Generate the fast path, if possible.
- PreserveJVMState pjvms(this);
- generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases,
- src, src_offset, dest, dest_offset,
- ConvI2X(copy_length), dest_uninitialized);
-
- // Present the results of the fast call.
- result_region->init_req(fast_path, control());
- result_i_o ->init_req(fast_path, i_o());
- result_memory->init_req(fast_path, memory(adr_type));
+ if (stopped()) {
+ return true;
}
- // Here are all the slow paths up to this point, in one bundle:
- slow_control = top();
- if (slow_region != NULL)
- slow_control = _gvn.transform(slow_region);
- DEBUG_ONLY(slow_region = (RegionNode*)badAddress);
-
- set_control(checked_control);
- if (!stopped()) {
- // Clean up after the checked call.
- // The returned value is either 0 or -1^K,
- // where K = number of partially transferred array elements.
- Node* cmp = _gvn.transform(new CmpINode(checked_value, intcon(0)));
- Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq));
- IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
-
- // If it is 0, we are done, so transfer to the end.
- Node* checks_done = _gvn.transform(new IfTrueNode(iff));
- result_region->init_req(checked_path, checks_done);
- result_i_o ->init_req(checked_path, checked_i_o);
- result_memory->init_req(checked_path, checked_mem);
-
- // If it is not zero, merge into the slow call.
- set_control( _gvn.transform(new IfFalseNode(iff) ));
- RegionNode* slow_reg2 = new RegionNode(3);
- PhiNode* slow_i_o2 = new PhiNode(slow_reg2, Type::ABIO);
- PhiNode* slow_mem2 = new PhiNode(slow_reg2, Type::MEMORY, adr_type);
- record_for_igvn(slow_reg2);
- slow_reg2 ->init_req(1, slow_control);
- slow_i_o2 ->init_req(1, slow_i_o);
- slow_mem2 ->init_req(1, slow_mem);
- slow_reg2 ->init_req(2, control());
- slow_i_o2 ->init_req(2, checked_i_o);
- slow_mem2 ->init_req(2, checked_mem);
-
- slow_control = _gvn.transform(slow_reg2);
- slow_i_o = _gvn.transform(slow_i_o2);
- slow_mem = _gvn.transform(slow_mem2);
-
- if (alloc != NULL) {
- // We'll restart from the very beginning, after zeroing the whole thing.
- // This can cause double writes, but that's OK since dest is brand new.
- // So we ignore the low 31 bits of the value returned from the stub.
- } else {
- // We must continue the copy exactly where it failed, or else
- // another thread might see the wrong number of writes to dest.
- Node* checked_offset = _gvn.transform(new XorINode(checked_value, intcon(-1)));
- Node* slow_offset = new PhiNode(slow_reg2, TypeInt::INT);
- slow_offset->init_req(1, intcon(0));
- slow_offset->init_req(2, checked_offset);
- slow_offset = _gvn.transform(slow_offset);
-
- // Adjust the arguments by the conditionally incoming offset.
- Node* src_off_plus = _gvn.transform(new AddINode(src_offset, slow_offset));
- Node* dest_off_plus = _gvn.transform(new AddINode(dest_offset, slow_offset));
- Node* length_minus = _gvn.transform(new SubINode(copy_length, slow_offset));
-
- // Tweak the node variables to adjust the code produced below:
- src_offset = src_off_plus;
- dest_offset = dest_off_plus;
- copy_length = length_minus;
- }
+ AllocateArrayNode* alloc = tightly_coupled_allocation(dest, NULL);
+ ArrayCopyNode* ac = ArrayCopyNode::make(this, true, src, src_offset, dest, dest_offset, length, alloc != NULL,
+ // Create LoadRange and LoadKlass nodes for use during macro expansion here
+ // so the compiler has a chance to eliminate them: during macro expansion,
+ // we have to set their control (CastPP nodes are eliminated).
+ load_array_length(src), load_array_length(dest),
+ load_object_klass(src), load_object_klass(dest));
+
+ if (notest) {
+ ac->set_arraycopy_notest();
}
- set_control(slow_control);
- if (!stopped()) {
- // Generate the slow path, if needed.
- PreserveJVMState pjvms(this); // replace_in_map may trash the map
-
- set_memory(slow_mem, adr_type);
- set_i_o(slow_i_o);
-
- if (dest_uninitialized) {
- generate_clear_array(adr_type, dest, basic_elem_type,
- intcon(0), NULL,
- alloc->in(AllocateNode::AllocSize));
- }
-
- generate_slow_arraycopy(adr_type,
- src, src_offset, dest, dest_offset,
- copy_length, /*dest_uninitialized*/false);
-
- result_region->init_req(slow_call_path, control());
- result_i_o ->init_req(slow_call_path, i_o());
- result_memory->init_req(slow_call_path, memory(adr_type));
- }
-
- // Remove unused edges.
- for (uint i = 1; i < result_region->req(); i++) {
- if (result_region->in(i) == NULL)
- result_region->init_req(i, top());
- }
-
- // Finished; return the combined state.
- set_control( _gvn.transform(result_region));
- set_i_o( _gvn.transform(result_i_o) );
- set_memory( _gvn.transform(result_memory), adr_type );
-
- // The memory edges above are precise in order to model effects around
- // array copies accurately to allow value numbering of field loads around
- // arraycopy. Such field loads, both before and after, are common in Java
- // collections and similar classes involving header/array data structures.
- //
- // But with low number of register or when some registers are used or killed
- // by arraycopy calls it causes registers spilling on stack. See 6544710.
- // The next memory barrier is added to avoid it. If the arraycopy can be
- // optimized away (which it can, sometimes) then we can manually remove
- // the membar also.
- //
- // Do not let reads from the cloned object float above the arraycopy.
- if (alloc != NULL) {
- // Do not let stores that initialize this object be reordered with
- // a subsequent store that would make this object accessible by
- // other threads.
- // Record what AllocateNode this StoreStore protects so that
- // escape analysis can go from the MemBarStoreStoreNode to the
- // AllocateNode and eliminate the MemBarStoreStoreNode if possible
- // based on the escape status of the AllocateNode.
- insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
- } else if (InsertMemBarAfterArraycopy)
- insert_mem_bar(Op_MemBarCPUOrder);
+ Node* n = _gvn.transform(ac);
+ assert(n == ac, "cannot disappear");
+ ac->connect_outputs(this);
+
+ return true;
}
@@ -5397,305 +4883,6 @@
return alloc;
}
-// Helper for initialization of arrays, creating a ClearArray.
-// It writes zero bits in [start..end), within the body of an array object.
-// The memory effects are all chained onto the 'adr_type' alias category.
-//
-// Since the object is otherwise uninitialized, we are free
-// to put a little "slop" around the edges of the cleared area,
-// as long as it does not go back into the array's header,
-// or beyond the array end within the heap.
-//
-// The lower edge can be rounded down to the nearest jint and the
-// upper edge can be rounded up to the nearest MinObjAlignmentInBytes.
-//
-// Arguments:
-// adr_type memory slice where writes are generated
-// dest oop of the destination array
-// basic_elem_type element type of the destination
-// slice_idx array index of first element to store
-// slice_len number of elements to store (or NULL)
-// dest_size total size in bytes of the array object
-//
-// Exactly one of slice_len or dest_size must be non-NULL.
-// If dest_size is non-NULL, zeroing extends to the end of the object.
-// If slice_len is non-NULL, the slice_idx value must be a constant.
-void
-LibraryCallKit::generate_clear_array(const TypePtr* adr_type,
- Node* dest,
- BasicType basic_elem_type,
- Node* slice_idx,
- Node* slice_len,
- Node* dest_size) {
- // one or the other but not both of slice_len and dest_size:
- assert((slice_len != NULL? 1: 0) + (dest_size != NULL? 1: 0) == 1, "");
- if (slice_len == NULL) slice_len = top();
- if (dest_size == NULL) dest_size = top();
-
- // operate on this memory slice:
- Node* mem = memory(adr_type); // memory slice to operate on
-
- // scaling and rounding of indexes:
- int scale = exact_log2(type2aelembytes(basic_elem_type));
- int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
- int clear_low = (-1 << scale) & (BytesPerInt - 1);
- int bump_bit = (-1 << scale) & BytesPerInt;
-
- // determine constant starts and ends
- const intptr_t BIG_NEG = -128;
- assert(BIG_NEG + 2*abase < 0, "neg enough");
- intptr_t slice_idx_con = (intptr_t) find_int_con(slice_idx, BIG_NEG);
- intptr_t slice_len_con = (intptr_t) find_int_con(slice_len, BIG_NEG);
- if (slice_len_con == 0) {
- return; // nothing to do here
- }
- intptr_t start_con = (abase + (slice_idx_con << scale)) & ~clear_low;
- intptr_t end_con = find_intptr_t_con(dest_size, -1);
- if (slice_idx_con >= 0 && slice_len_con >= 0) {
- assert(end_con < 0, "not two cons");
- end_con = round_to(abase + ((slice_idx_con + slice_len_con) << scale),
- BytesPerLong);
- }
-
- if (start_con >= 0 && end_con >= 0) {
- // Constant start and end. Simple.
- mem = ClearArrayNode::clear_memory(control(), mem, dest,
- start_con, end_con, &_gvn);
- } else if (start_con >= 0 && dest_size != top()) {
- // Constant start, pre-rounded end after the tail of the array.
- Node* end = dest_size;
- mem = ClearArrayNode::clear_memory(control(), mem, dest,
- start_con, end, &_gvn);
- } else if (start_con >= 0 && slice_len != top()) {
- // Constant start, non-constant end. End needs rounding up.
- // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8)
- intptr_t end_base = abase + (slice_idx_con << scale);
- int end_round = (-1 << scale) & (BytesPerLong - 1);
- Node* end = ConvI2X(slice_len);
- if (scale != 0)
- end = _gvn.transform(new LShiftXNode(end, intcon(scale) ));
- end_base += end_round;
- end = _gvn.transform(new AddXNode(end, MakeConX(end_base)));
- end = _gvn.transform(new AndXNode(end, MakeConX(~end_round)));
- mem = ClearArrayNode::clear_memory(control(), mem, dest,
- start_con, end, &_gvn);
- } else if (start_con < 0 && dest_size != top()) {
- // Non-constant start, pre-rounded end after the tail of the array.
- // This is almost certainly a "round-to-end" operation.
- Node* start = slice_idx;
- start = ConvI2X(start);
- if (scale != 0)
- start = _gvn.transform(new LShiftXNode( start, intcon(scale) ));
- start = _gvn.transform(new AddXNode(start, MakeConX(abase)));
- if ((bump_bit | clear_low) != 0) {
- int to_clear = (bump_bit | clear_low);
- // Align up mod 8, then store a jint zero unconditionally
- // just before the mod-8 boundary.
- if (((abase + bump_bit) & ~to_clear) - bump_bit
- < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) {
- bump_bit = 0;
- assert((abase & to_clear) == 0, "array base must be long-aligned");
- } else {
- // Bump 'start' up to (or past) the next jint boundary:
- start = _gvn.transform(new AddXNode(start, MakeConX(bump_bit)));
- assert((abase & clear_low) == 0, "array base must be int-aligned");
- }
- // Round bumped 'start' down to jlong boundary in body of array.
- start = _gvn.transform(new AndXNode(start, MakeConX(~to_clear)));
- if (bump_bit != 0) {
- // Store a zero to the immediately preceding jint:
- Node* x1 = _gvn.transform(new AddXNode(start, MakeConX(-bump_bit)));
- Node* p1 = basic_plus_adr(dest, x1);
- mem = StoreNode::make(_gvn, control(), mem, p1, adr_type, intcon(0), T_INT, MemNode::unordered);
- mem = _gvn.transform(mem);
- }
- }
- Node* end = dest_size; // pre-rounded
- mem = ClearArrayNode::clear_memory(control(), mem, dest,
- start, end, &_gvn);
- } else {
- // Non-constant start, unrounded non-constant end.
- // (Nobody zeroes a random midsection of an array using this routine.)
- ShouldNotReachHere(); // fix caller
- }
-
- // Done.
- set_memory(mem, adr_type);
-}
-
-
-bool
-LibraryCallKit::generate_block_arraycopy(const TypePtr* adr_type,
- BasicType basic_elem_type,
- AllocateNode* alloc,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* dest_size, bool dest_uninitialized) {
- // See if there is an advantage from block transfer.
- int scale = exact_log2(type2aelembytes(basic_elem_type));
- if (scale >= LogBytesPerLong)
- return false; // it is already a block transfer
-
- // Look at the alignment of the starting offsets.
- int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
-
- intptr_t src_off_con = (intptr_t) find_int_con(src_offset, -1);
- intptr_t dest_off_con = (intptr_t) find_int_con(dest_offset, -1);
- if (src_off_con < 0 || dest_off_con < 0)
- // At present, we can only understand constants.
- return false;
-
- intptr_t src_off = abase + (src_off_con << scale);
- intptr_t dest_off = abase + (dest_off_con << scale);
-
- if (((src_off | dest_off) & (BytesPerLong-1)) != 0) {
- // Non-aligned; too bad.
- // One more chance: Pick off an initial 32-bit word.
- // This is a common case, since abase can be odd mod 8.
- if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt &&
- ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) {
- Node* sptr = basic_plus_adr(src, src_off);
- Node* dptr = basic_plus_adr(dest, dest_off);
- Node* sval = make_load(control(), sptr, TypeInt::INT, T_INT, adr_type, MemNode::unordered);
- store_to_memory(control(), dptr, sval, T_INT, adr_type, MemNode::unordered);
- src_off += BytesPerInt;
- dest_off += BytesPerInt;
- } else {
- return false;
- }
- }
- assert(src_off % BytesPerLong == 0, "");
- assert(dest_off % BytesPerLong == 0, "");
-
- // Do this copy by giant steps.
- Node* sptr = basic_plus_adr(src, src_off);
- Node* dptr = basic_plus_adr(dest, dest_off);
- Node* countx = dest_size;
- countx = _gvn.transform(new SubXNode(countx, MakeConX(dest_off)));
- countx = _gvn.transform(new URShiftXNode(countx, intcon(LogBytesPerLong)));
-
- bool disjoint_bases = true; // since alloc != NULL
- generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
- sptr, NULL, dptr, NULL, countx, dest_uninitialized);
-
- return true;
-}
-
-
-// Helper function; generates code for the slow case.
-// We make a call to a runtime method which emulates the native method,
-// but without the native wrapper overhead.
-void
-LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length, bool dest_uninitialized) {
- assert(!dest_uninitialized, "Invariant");
- Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
- OptoRuntime::slow_arraycopy_Type(),
- OptoRuntime::slow_arraycopy_Java(),
- "slow_arraycopy", adr_type,
- src, src_offset, dest, dest_offset,
- copy_length);
-
- // Handle exceptions thrown by this fellow:
- make_slow_call_ex(call, env()->Throwable_klass(), false);
-}
-
-// Helper function; generates code for cases requiring runtime checks.
-Node*
-LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type,
- Node* dest_elem_klass,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length, bool dest_uninitialized) {
- if (stopped()) return NULL;
-
- address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized);
- if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
- return NULL;
- }
-
- // Pick out the parameters required to perform a store-check
- // for the target array. This is an optimistic check. It will
- // look in each non-null element's class, at the desired klass's
- // super_check_offset, for the desired klass.
- int sco_offset = in_bytes(Klass::super_check_offset_offset());
- Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset);
- Node* n3 = new LoadINode(NULL, memory(p3), p3, _gvn.type(p3)->is_ptr(), TypeInt::INT, MemNode::unordered);
- Node* check_offset = ConvI2X(_gvn.transform(n3));
- Node* check_value = dest_elem_klass;
-
- Node* src_start = array_element_address(src, src_offset, T_OBJECT);
- Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT);
-
- // (We know the arrays are never conjoint, because their types differ.)
- Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
- OptoRuntime::checkcast_arraycopy_Type(),
- copyfunc_addr, "checkcast_arraycopy", adr_type,
- // five arguments, of which two are
- // intptr_t (jlong in LP64)
- src_start, dest_start,
- copy_length XTOP,
- check_offset XTOP,
- check_value);
-
- return _gvn.transform(new ProjNode(call, TypeFunc::Parms));
-}
-
-
-// Helper function; generates code for cases requiring runtime checks.
-Node*
-LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length, bool dest_uninitialized) {
- assert(!dest_uninitialized, "Invariant");
- if (stopped()) return NULL;
- address copyfunc_addr = StubRoutines::generic_arraycopy();
- if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
- return NULL;
- }
-
- Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
- OptoRuntime::generic_arraycopy_Type(),
- copyfunc_addr, "generic_arraycopy", adr_type,
- src, src_offset, dest, dest_offset, copy_length);
-
- return _gvn.transform(new ProjNode(call, TypeFunc::Parms));
-}
-
-// Helper function; generates the fast out-of-line call to an arraycopy stub.
-void
-LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type,
- BasicType basic_elem_type,
- bool disjoint_bases,
- Node* src, Node* src_offset,
- Node* dest, Node* dest_offset,
- Node* copy_length, bool dest_uninitialized) {
- if (stopped()) return; // nothing to do
-
- Node* src_start = src;
- Node* dest_start = dest;
- if (src_offset != NULL || dest_offset != NULL) {
- assert(src_offset != NULL && dest_offset != NULL, "");
- src_start = array_element_address(src, src_offset, basic_elem_type);
- dest_start = array_element_address(dest, dest_offset, basic_elem_type);
- }
-
- // Figure out which arraycopy runtime method to call.
- const char* copyfunc_name = "arraycopy";
- address copyfunc_addr =
- basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
- disjoint_bases, copyfunc_name, dest_uninitialized);
-
- // Call it. Note that the count_ix value is not scaled to a byte-size.
- make_runtime_call(RC_LEAF|RC_NO_FP,
- OptoRuntime::fast_arraycopy_Type(),
- copyfunc_addr, copyfunc_name, adr_type,
- src_start, dest_start, copy_length XTOP);
-}
-
//-------------inline_encodeISOArray-----------------------------------
// encode char[] to byte[] in ISO_8859_1
bool LibraryCallKit::inline_encodeISOArray() {
@@ -5915,8 +5102,19 @@
type = Type::get_const_basic_type(bt);
}
+ if (support_IRIW_for_not_multiple_copy_atomic_cpu && is_vol) {
+ insert_mem_bar(Op_MemBarVolatile); // StoreLoad barrier
+ }
// Build the load.
- Node* loadedField = make_load(NULL, adr, type, bt, adr_type, MemNode::unordered, is_vol);
+ MemNode::MemOrd mo = is_vol ? MemNode::acquire : MemNode::unordered;
+ Node* loadedField = make_load(NULL, adr, type, bt, adr_type, mo, is_vol);
+ // If reference is volatile, prevent following memory ops from
+ // floating up past the volatile read. Also prevents commoning
+ // another volatile read.
+ if (is_vol) {
+ // Memory barrier includes bogus read of value to force load BEFORE membar
+ insert_mem_bar(Op_MemBarAcquire, loadedField);
+ }
return loadedField;
}
--- a/hotspot/src/share/vm/opto/loopPredicate.cpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/opto/loopPredicate.cpp Thu Aug 21 19:06:21 2014 +0000
@@ -763,9 +763,7 @@
loop->dump_head();
}
#endif
- } else if ((cl != NULL) && (proj->_con == predicate_proj->_con) &&
- loop->is_range_check_if(iff, this, invar)) {
-
+ } else if (cl != NULL && loop->is_range_check_if(iff, this, invar)) {
// Range check for counted loops
const Node* cmp = bol->in(1)->as_Cmp();
Node* idx = cmp->in(1);
@@ -800,18 +798,31 @@
}
// Test the lower bound
- Node* lower_bound_bol = rc_predicate(loop, ctrl, scale, offset, init, limit, stride, rng, false);
+ BoolNode* lower_bound_bol = rc_predicate(loop, ctrl, scale, offset, init, limit, stride, rng, false);
+ // Negate test if necessary
+ bool negated = false;
+ if (proj->_con != predicate_proj->_con) {
+ lower_bound_bol = new BoolNode(lower_bound_bol->in(1), lower_bound_bol->_test.negate());
+ register_new_node(lower_bound_bol, ctrl);
+ negated = true;
+ }
IfNode* lower_bound_iff = lower_bound_proj->in(0)->as_If();
_igvn.hash_delete(lower_bound_iff);
lower_bound_iff->set_req(1, lower_bound_bol);
- if (TraceLoopPredicate) tty->print_cr("lower bound check if: %d", lower_bound_iff->_idx);
+ if (TraceLoopPredicate) tty->print_cr("lower bound check if: %s %d ", negated ? " negated" : "", lower_bound_iff->_idx);
// Test the upper bound
- Node* upper_bound_bol = rc_predicate(loop, lower_bound_proj, scale, offset, init, limit, stride, rng, true);
+ BoolNode* upper_bound_bol = rc_predicate(loop, lower_bound_proj, scale, offset, init, limit, stride, rng, true);
+ negated = false;
+ if (proj->_con != predicate_proj->_con) {
+ upper_bound_bol = new BoolNode(upper_bound_bol->in(1), upper_bound_bol->_test.negate());
+ register_new_node(upper_bound_bol, ctrl);
+ negated = true;
+ }
IfNode* upper_bound_iff = upper_bound_proj->in(0)->as_If();
_igvn.hash_delete(upper_bound_iff);
upper_bound_iff->set_req(1, upper_bound_bol);
- if (TraceLoopPredicate) tty->print_cr("upper bound check if: %d", lower_bound_iff->_idx);
+ if (TraceLoopPredicate) tty->print_cr("upper bound check if: %s %d ", negated ? " negated" : "", lower_bound_iff->_idx);
// Fall through into rest of the clean up code which will move
// any dependent nodes onto the upper bound test.
--- a/hotspot/src/share/vm/opto/macro.cpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/opto/macro.cpp Thu Aug 21 19:06:21 2014 +0000
@@ -94,7 +94,8 @@
newcall->add_req(old_in);
}
- newcall->set_jvms(oldcall->jvms());
+ // JVMS may be shared so clone it before we modify it
+ newcall->set_jvms(oldcall->jvms() != NULL ? oldcall->jvms()->clone_deep(C) : NULL);
for (JVMState *jvms = newcall->jvms(); jvms != NULL; jvms = jvms->caller()) {
jvms->set_map(newcall);
jvms->set_locoff(jvms->locoff()+jvms_adj);
@@ -2469,6 +2470,8 @@
assert(!n->as_AbstractLock()->is_eliminated(), "sanity");
_has_locks = true;
break;
+ case Node::Class_ArrayCopy:
+ break;
default:
assert(n->Opcode() == Op_LoopLimit ||
n->Opcode() == Op_Opaque1 ||
@@ -2544,6 +2547,25 @@
}
}
+ // expand arraycopy "macro" nodes first
+ // For ReduceBulkZeroing, we must first process all arraycopy nodes
+ // before the allocate nodes are expanded.
+ int macro_idx = C->macro_count() - 1;
+ while (macro_idx >= 0) {
+ Node * n = C->macro_node(macro_idx);
+ assert(n->is_macro(), "only macro nodes expected here");
+ if (_igvn.type(n) == Type::TOP || n->in(0)->is_top() ) {
+ // node is unreachable, so don't try to expand it
+ C->remove_macro_node(n);
+ } else if (n->is_ArrayCopy()){
+ int macro_count = C->macro_count();
+ expand_arraycopy_node(n->as_ArrayCopy());
+ assert(C->macro_count() < macro_count, "must have deleted a node from macro list");
+ }
+ if (C->failing()) return true;
+ macro_idx --;
+ }
+
// expand "macro" nodes
// nodes are removed from the macro list as they are processed
while (C->macro_count() > 0) {
--- a/hotspot/src/share/vm/opto/macro.hpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/opto/macro.hpp Thu Aug 21 19:06:21 2014 +0000
@@ -37,7 +37,7 @@
private:
PhaseIterGVN &_igvn;
- // Helper methods roughly modelled after GraphKit:
+ // Helper methods roughly modeled after GraphKit:
Node* top() const { return C->top(); }
Node* intcon(jint con) const { return _igvn.intcon(con); }
Node* longcon(jlong con) const { return _igvn.longcon(con); }
@@ -101,6 +101,86 @@
void expand_lock_node(LockNode *lock);
void expand_unlock_node(UnlockNode *unlock);
+ // More helper methods modeled after GraphKit for array copy
+ void insert_mem_bar(Node** ctrl, Node** mem, int opcode, Node* precedent = NULL);
+ Node* array_element_address(Node* ary, Node* idx, BasicType elembt);
+ Node* ConvI2L(Node* offset);
+ Node* make_leaf_call(Node* ctrl, Node* mem,
+ const TypeFunc* call_type, address call_addr,
+ const char* call_name,
+ const TypePtr* adr_type,
+ Node* parm0 = NULL, Node* parm1 = NULL,
+ Node* parm2 = NULL, Node* parm3 = NULL,
+ Node* parm4 = NULL, Node* parm5 = NULL,
+ Node* parm6 = NULL, Node* parm7 = NULL);
+
+ // helper methods modeled after LibraryCallKit for array copy
+ Node* generate_guard(Node** ctrl, Node* test, RegionNode* region, float true_prob);
+ Node* generate_slow_guard(Node** ctrl, Node* test, RegionNode* region);
+ void generate_negative_guard(Node** ctrl, Node* index, RegionNode* region);
+ void generate_limit_guard(Node** ctrl, Node* offset, Node* subseq_length, Node* array_length, RegionNode* region);
+
+ // More helper methods for array copy
+ Node* generate_nonpositive_guard(Node** ctrl, Node* index, bool never_negative);
+ void finish_arraycopy_call(Node* call, Node** ctrl, MergeMemNode** mem, const TypePtr* adr_type);
+ address basictype2arraycopy(BasicType t,
+ Node* src_offset,
+ Node* dest_offset,
+ bool disjoint_bases,
+ const char* &name,
+ bool dest_uninitialized);
+ Node* generate_arraycopy(ArrayCopyNode *ac,
+ AllocateArrayNode* alloc,
+ Node** ctrl, MergeMemNode* mem, Node** io,
+ const TypePtr* adr_type,
+ BasicType basic_elem_type,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length,
+ bool disjoint_bases = false,
+ bool length_never_negative = false,
+ RegionNode* slow_region = NULL);
+ void generate_clear_array(Node* ctrl, MergeMemNode* merge_mem,
+ const TypePtr* adr_type,
+ Node* dest,
+ BasicType basic_elem_type,
+ Node* slice_idx,
+ Node* slice_len,
+ Node* dest_size);
+ bool generate_block_arraycopy(Node** ctrl, MergeMemNode** mem, Node* io,
+ const TypePtr* adr_type,
+ BasicType basic_elem_type,
+ AllocateNode* alloc,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* dest_size, bool dest_uninitialized);
+ MergeMemNode* generate_slow_arraycopy(ArrayCopyNode *ac,
+ Node** ctrl, Node* mem, Node** io,
+ const TypePtr* adr_type,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length, bool dest_uninitialized);
+ Node* generate_checkcast_arraycopy(Node** ctrl, MergeMemNode** mem,
+ const TypePtr* adr_type,
+ Node* dest_elem_klass,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length, bool dest_uninitialized);
+ Node* generate_generic_arraycopy(Node** ctrl, MergeMemNode** mem,
+ const TypePtr* adr_type,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length, bool dest_uninitialized);
+ void generate_unchecked_arraycopy(Node** ctrl, MergeMemNode** mem,
+ const TypePtr* adr_type,
+ BasicType basic_elem_type,
+ bool disjoint_bases,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length, bool dest_uninitialized);
+
+ void expand_arraycopy_node(ArrayCopyNode *ac);
+
int replace_input(Node *use, Node *oldref, Node *newref);
void copy_call_debug_info(CallNode *oldcall, CallNode * newcall);
Node* opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path = false);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hotspot/src/share/vm/opto/macroArrayCopy.cpp Thu Aug 21 19:06:21 2014 +0000
@@ -0,0 +1,1233 @@
+/*
+ * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "oops/objArrayKlass.hpp"
+#include "opto/convertnode.hpp"
+#include "opto/graphKit.hpp"
+#include "opto/macro.hpp"
+#include "opto/runtime.hpp"
+
+
+void PhaseMacroExpand::insert_mem_bar(Node** ctrl, Node** mem, int opcode, Node* precedent) {
+ MemBarNode* mb = MemBarNode::make(C, opcode, Compile::AliasIdxBot, precedent);
+ mb->init_req(TypeFunc::Control, *ctrl);
+ mb->init_req(TypeFunc::Memory, *mem);
+ transform_later(mb);
+ *ctrl = new ProjNode(mb,TypeFunc::Control);
+ transform_later(*ctrl);
+ Node* mem_proj = new ProjNode(mb,TypeFunc::Memory);
+ transform_later(mem_proj);
+ *mem = mem_proj;
+}
+
+Node* PhaseMacroExpand::array_element_address(Node* ary, Node* idx, BasicType elembt) {
+ uint shift = exact_log2(type2aelembytes(elembt));
+ uint header = arrayOopDesc::base_offset_in_bytes(elembt);
+ Node* base = basic_plus_adr(ary, header);
+#ifdef _LP64
+ // see comment in GraphKit::array_element_address
+ int index_max = max_jint - 1; // array size is max_jint, index is one less
+ const TypeLong* lidxtype = TypeLong::make(CONST64(0), index_max, Type::WidenMax);
+ idx = transform_later( new ConvI2LNode(idx, lidxtype) );
+#endif
+ Node* scale = new LShiftXNode(idx, intcon(shift));
+ transform_later(scale);
+ return basic_plus_adr(ary, base, scale);
+}
+
+Node* PhaseMacroExpand::ConvI2L(Node* offset) {
+ return transform_later(new ConvI2LNode(offset));
+}
+
+Node* PhaseMacroExpand::make_leaf_call(Node* ctrl, Node* mem,
+ const TypeFunc* call_type, address call_addr,
+ const char* call_name,
+ const TypePtr* adr_type,
+ Node* parm0, Node* parm1,
+ Node* parm2, Node* parm3,
+ Node* parm4, Node* parm5,
+ Node* parm6, Node* parm7) {
+ int size = call_type->domain()->cnt();
+ Node* call = new CallLeafNoFPNode(call_type, call_addr, call_name, adr_type);
+ call->init_req(TypeFunc::Control, ctrl);
+ call->init_req(TypeFunc::I_O , top());
+ call->init_req(TypeFunc::Memory , mem);
+ call->init_req(TypeFunc::ReturnAdr, top());
+ call->init_req(TypeFunc::FramePtr, top());
+
+ // Hook each parm in order. Stop looking at the first NULL.
+ if (parm0 != NULL) { call->init_req(TypeFunc::Parms+0, parm0);
+ if (parm1 != NULL) { call->init_req(TypeFunc::Parms+1, parm1);
+ if (parm2 != NULL) { call->init_req(TypeFunc::Parms+2, parm2);
+ if (parm3 != NULL) { call->init_req(TypeFunc::Parms+3, parm3);
+ if (parm4 != NULL) { call->init_req(TypeFunc::Parms+4, parm4);
+ if (parm5 != NULL) { call->init_req(TypeFunc::Parms+5, parm5);
+ if (parm6 != NULL) { call->init_req(TypeFunc::Parms+6, parm6);
+ if (parm7 != NULL) { call->init_req(TypeFunc::Parms+7, parm7);
+ /* close each nested if ===> */ } } } } } } } }
+ assert(call->in(call->req()-1) != NULL, "must initialize all parms");
+
+ return call;
+}
+
+
+//------------------------------generate_guard---------------------------
+// Helper function for generating guarded fast-slow graph structures.
+// The given 'test', if true, guards a slow path. If the test fails
+// then a fast path can be taken. (We generally hope it fails.)
+// In all cases, GraphKit::control() is updated to the fast path.
+// The returned value represents the control for the slow path.
+// The return value is never 'top'; it is either a valid control
+// or NULL if it is obvious that the slow path can never be taken.
+// Also, if region and the slow control are not NULL, the slow edge
+// is appended to the region.
+Node* PhaseMacroExpand::generate_guard(Node** ctrl, Node* test, RegionNode* region, float true_prob) {
+ if ((*ctrl)->is_top()) {
+ // Already short circuited.
+ return NULL;
+ }
+ // Build an if node and its projections.
+ // If test is true we take the slow path, which we assume is uncommon.
+ if (_igvn.type(test) == TypeInt::ZERO) {
+ // The slow branch is never taken. No need to build this guard.
+ return NULL;
+ }
+
+ IfNode* iff = new IfNode(*ctrl, test, true_prob, COUNT_UNKNOWN);
+ transform_later(iff);
+
+ Node* if_slow = new IfTrueNode(iff);
+ transform_later(if_slow);
+
+ if (region != NULL) {
+ region->add_req(if_slow);
+ }
+
+ Node* if_fast = new IfFalseNode(iff);
+ transform_later(if_fast);
+
+ *ctrl = if_fast;
+
+ return if_slow;
+}
+
+inline Node* PhaseMacroExpand::generate_slow_guard(Node** ctrl, Node* test, RegionNode* region) {
+ return generate_guard(ctrl, test, region, PROB_UNLIKELY_MAG(3));
+}
+
+void PhaseMacroExpand::generate_negative_guard(Node** ctrl, Node* index, RegionNode* region) {
+ if ((*ctrl)->is_top())
+ return; // already stopped
+ if (_igvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
+ return; // index is already adequately typed
+ Node* cmp_lt = new CmpINode(index, intcon(0));
+ transform_later(cmp_lt);
+ Node* bol_lt = new BoolNode(cmp_lt, BoolTest::lt);
+ transform_later(bol_lt);
+ generate_guard(ctrl, bol_lt, region, PROB_MIN);
+}
+
+void PhaseMacroExpand::generate_limit_guard(Node** ctrl, Node* offset, Node* subseq_length, Node* array_length, RegionNode* region) {
+ if ((*ctrl)->is_top())
+ return; // already stopped
+ bool zero_offset = _igvn.type(offset) == TypeInt::ZERO;
+ if (zero_offset && subseq_length->eqv_uncast(array_length))
+ return; // common case of whole-array copy
+ Node* last = subseq_length;
+ if (!zero_offset) { // last += offset
+ last = new AddINode(last, offset);
+ transform_later(last);
+ }
+ Node* cmp_lt = new CmpUNode(array_length, last);
+ transform_later(cmp_lt);
+ Node* bol_lt = new BoolNode(cmp_lt, BoolTest::lt);
+ transform_later(bol_lt);
+ generate_guard(ctrl, bol_lt, region, PROB_MIN);
+}
+
+Node* PhaseMacroExpand::generate_nonpositive_guard(Node** ctrl, Node* index, bool never_negative) {
+ if ((*ctrl)->is_top()) return NULL;
+
+ if (_igvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
+ return NULL; // index is already adequately typed
+ Node* cmp_le = new CmpINode(index, intcon(0));
+ transform_later(cmp_le);
+ BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
+ Node* bol_le = new BoolNode(cmp_le, le_or_eq);
+ transform_later(bol_le);
+ Node* is_notp = generate_guard(ctrl, bol_le, NULL, PROB_MIN);
+
+ return is_notp;
+}
+
+void PhaseMacroExpand::finish_arraycopy_call(Node* call, Node** ctrl, MergeMemNode** mem, const TypePtr* adr_type) {
+ transform_later(call);
+
+ *ctrl = new ProjNode(call,TypeFunc::Control);
+ transform_later(*ctrl);
+ Node* newmem = new ProjNode(call, TypeFunc::Memory);
+ transform_later(newmem);
+
+ uint alias_idx = C->get_alias_index(adr_type);
+ if (alias_idx != Compile::AliasIdxBot) {
+ *mem = MergeMemNode::make(*mem);
+ (*mem)->set_memory_at(alias_idx, newmem);
+ } else {
+ *mem = MergeMemNode::make(newmem);
+ }
+ transform_later(*mem);
+}
+
+address PhaseMacroExpand::basictype2arraycopy(BasicType t,
+ Node* src_offset,
+ Node* dest_offset,
+ bool disjoint_bases,
+ const char* &name,
+ bool dest_uninitialized) {
+ const TypeInt* src_offset_inttype = _igvn.find_int_type(src_offset);;
+ const TypeInt* dest_offset_inttype = _igvn.find_int_type(dest_offset);;
+
+ bool aligned = false;
+ bool disjoint = disjoint_bases;
+
+ // if the offsets are the same, we can treat the memory regions as
+ // disjoint, because either the memory regions are in different arrays,
+ // or they are identical (which we can treat as disjoint.) We can also
+ // treat a copy with a destination index less that the source index
+ // as disjoint since a low->high copy will work correctly in this case.
+ if (src_offset_inttype != NULL && src_offset_inttype->is_con() &&
+ dest_offset_inttype != NULL && dest_offset_inttype->is_con()) {
+ // both indices are constants
+ int s_offs = src_offset_inttype->get_con();
+ int d_offs = dest_offset_inttype->get_con();
+ int element_size = type2aelembytes(t);
+ aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) &&
+ ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0);
+ if (s_offs >= d_offs) disjoint = true;
+ } else if (src_offset == dest_offset && src_offset != NULL) {
+ // This can occur if the offsets are identical non-constants.
+ disjoint = true;
+ }
+
+ return StubRoutines::select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized);
+}
+
+#define COMMA ,
+#define XTOP LP64_ONLY(COMMA top())
+
+// Generate an optimized call to arraycopy.
+// Caller must guard against non-arrays.
+// Caller must determine a common array basic-type for both arrays.
+// Caller must validate offsets against array bounds.
+// The slow_region has already collected guard failure paths
+// (such as out of bounds length or non-conformable array types).
+// The generated code has this shape, in general:
+//
+// if (length == 0) return // via zero_path
+// slowval = -1
+// if (types unknown) {
+// slowval = call generic copy loop
+// if (slowval == 0) return // via checked_path
+// } else if (indexes in bounds) {
+// if ((is object array) && !(array type check)) {
+// slowval = call checked copy loop
+// if (slowval == 0) return // via checked_path
+// } else {
+// call bulk copy loop
+// return // via fast_path
+// }
+// }
+// // adjust params for remaining work:
+// if (slowval != -1) {
+// n = -1^slowval; src_offset += n; dest_offset += n; length -= n
+// }
+// slow_region:
+// call slow arraycopy(src, src_offset, dest, dest_offset, length)
+// return // via slow_call_path
+//
+// This routine is used from several intrinsics: System.arraycopy,
+// Object.clone (the array subcase), and Arrays.copyOf[Range].
+//
+Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* alloc,
+ Node** ctrl, MergeMemNode* mem, Node** io,
+ const TypePtr* adr_type,
+ BasicType basic_elem_type,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length,
+ bool disjoint_bases,
+ bool length_never_negative,
+ RegionNode* slow_region) {
+ if (slow_region == NULL) {
+ slow_region = new RegionNode(1);
+ transform_later(slow_region);
+ }
+
+ Node* original_dest = dest;
+ bool dest_uninitialized = false;
+
+ // See if this is the initialization of a newly-allocated array.
+ // If so, we will take responsibility here for initializing it to zero.
+ // (Note: Because tightly_coupled_allocation performs checks on the
+ // out-edges of the dest, we need to avoid making derived pointers
+ // from it until we have checked its uses.)
+ if (ReduceBulkZeroing
+ && !ZeroTLAB // pointless if already zeroed
+ && basic_elem_type != T_CONFLICT // avoid corner case
+ && !src->eqv_uncast(dest)
+ && alloc != NULL
+ && _igvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0
+ && alloc->maybe_set_complete(&_igvn)) {
+ // "You break it, you buy it."
+ InitializeNode* init = alloc->initialization();
+ assert(init->is_complete(), "we just did this");
+ init->set_complete_with_arraycopy();
+ assert(dest->is_CheckCastPP(), "sanity");
+ assert(dest->in(0)->in(0) == init, "dest pinned");
+ adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory
+ // From this point on, every exit path is responsible for
+ // initializing any non-copied parts of the object to zero.
+ // Also, if this flag is set we make sure that arraycopy interacts properly
+ // with G1, eliding pre-barriers. See CR 6627983.
+ dest_uninitialized = true;
+ } else {
+ // No zeroing elimination here.
+ alloc = NULL;
+ //original_dest = dest;
+ //dest_uninitialized = false;
+ }
+
+ uint alias_idx = C->get_alias_index(adr_type);
+
+ // Results are placed here:
+ enum { fast_path = 1, // normal void-returning assembly stub
+ checked_path = 2, // special assembly stub with cleanup
+ slow_call_path = 3, // something went wrong; call the VM
+ zero_path = 4, // bypass when length of copy is zero
+ bcopy_path = 5, // copy primitive array by 64-bit blocks
+ PATH_LIMIT = 6
+ };
+ RegionNode* result_region = new RegionNode(PATH_LIMIT);
+ PhiNode* result_i_o = new PhiNode(result_region, Type::ABIO);
+ PhiNode* result_memory = new PhiNode(result_region, Type::MEMORY, adr_type);
+ assert(adr_type != TypePtr::BOTTOM, "must be RawMem or a T[] slice");
+ transform_later(result_region);
+ transform_later(result_i_o);
+ transform_later(result_memory);
+
+ // The slow_control path:
+ Node* slow_control;
+ Node* slow_i_o = *io;
+ Node* slow_mem = mem->memory_at(alias_idx);
+ DEBUG_ONLY(slow_control = (Node*) badAddress);
+
+ // Checked control path:
+ Node* checked_control = top();
+ Node* checked_mem = NULL;
+ Node* checked_i_o = NULL;
+ Node* checked_value = NULL;
+
+ if (basic_elem_type == T_CONFLICT) {
+ assert(!dest_uninitialized, "");
+ Node* cv = generate_generic_arraycopy(ctrl, &mem,
+ adr_type,
+ src, src_offset, dest, dest_offset,
+ copy_length, dest_uninitialized);
+ if (cv == NULL) cv = intcon(-1); // failure (no stub available)
+ checked_control = *ctrl;
+ checked_i_o = *io;
+ checked_mem = mem->memory_at(alias_idx);
+ checked_value = cv;
+ *ctrl = top();
+ }
+
+ Node* not_pos = generate_nonpositive_guard(ctrl, copy_length, length_never_negative);
+ if (not_pos != NULL) {
+ Node* local_ctrl = not_pos, *local_io = *io;
+ MergeMemNode* local_mem = MergeMemNode::make(mem);
+ transform_later(local_mem);
+
+ // (6) length must not be negative.
+ if (!length_never_negative) {
+ generate_negative_guard(&local_ctrl, copy_length, slow_region);
+ }
+
+ // copy_length is 0.
+ if (dest_uninitialized) {
+ assert(!local_ctrl->is_top(), "no ctrl?");
+ Node* dest_length = alloc->in(AllocateNode::ALength);
+ if (copy_length->eqv_uncast(dest_length)
+ || _igvn.find_int_con(dest_length, 1) <= 0) {
+ // There is no zeroing to do. No need for a secondary raw memory barrier.
+ } else {
+ // Clear the whole thing since there are no source elements to copy.
+ generate_clear_array(local_ctrl, local_mem,
+ adr_type, dest, basic_elem_type,
+ intcon(0), NULL,
+ alloc->in(AllocateNode::AllocSize));
+ // Use a secondary InitializeNode as raw memory barrier.
+ // Currently it is needed only on this path since other
+ // paths have stub or runtime calls as raw memory barriers.
+ MemBarNode* mb = MemBarNode::make(C, Op_Initialize,
+ Compile::AliasIdxRaw,
+ top());
+ transform_later(mb);
+ mb->set_req(TypeFunc::Control,local_ctrl);
+ mb->set_req(TypeFunc::Memory, local_mem->memory_at(Compile::AliasIdxRaw));
+ local_ctrl = transform_later(new ProjNode(mb, TypeFunc::Control));
+ local_mem->set_memory_at(Compile::AliasIdxRaw, transform_later(new ProjNode(mb, TypeFunc::Memory)));
+
+ InitializeNode* init = mb->as_Initialize();
+ init->set_complete(&_igvn); // (there is no corresponding AllocateNode)
+ }
+ }
+
+ // Present the results of the fast call.
+ result_region->init_req(zero_path, local_ctrl);
+ result_i_o ->init_req(zero_path, local_io);
+ result_memory->init_req(zero_path, local_mem->memory_at(alias_idx));
+ }
+
+ if (!(*ctrl)->is_top() && dest_uninitialized) {
+ // We have to initialize the *uncopied* part of the array to zero.
+ // The copy destination is the slice dest[off..off+len]. The other slices
+ // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
+ Node* dest_size = alloc->in(AllocateNode::AllocSize);
+ Node* dest_length = alloc->in(AllocateNode::ALength);
+ Node* dest_tail = transform_later( new AddINode(dest_offset, copy_length));
+
+ // If there is a head section that needs zeroing, do it now.
+ if (_igvn.find_int_con(dest_offset, -1) != 0) {
+ generate_clear_array(*ctrl, mem,
+ adr_type, dest, basic_elem_type,
+ intcon(0), dest_offset,
+ NULL);
+ }
+
+ // Next, perform a dynamic check on the tail length.
+ // It is often zero, and we can win big if we prove this.
+ // There are two wins: Avoid generating the ClearArray
+ // with its attendant messy index arithmetic, and upgrade
+ // the copy to a more hardware-friendly word size of 64 bits.
+ Node* tail_ctl = NULL;
+ if (!(*ctrl)->is_top() && !dest_tail->eqv_uncast(dest_length)) {
+ Node* cmp_lt = transform_later( new CmpINode(dest_tail, dest_length) );
+ Node* bol_lt = transform_later( new BoolNode(cmp_lt, BoolTest::lt) );
+ tail_ctl = generate_slow_guard(ctrl, bol_lt, NULL);
+ assert(tail_ctl != NULL || !(*ctrl)->is_top(), "must be an outcome");
+ }
+
+ // At this point, let's assume there is no tail.
+ if (!(*ctrl)->is_top() && alloc != NULL && basic_elem_type != T_OBJECT) {
+ // There is no tail. Try an upgrade to a 64-bit copy.
+ bool didit = false;
+ {
+ Node* local_ctrl = *ctrl, *local_io = *io;
+ MergeMemNode* local_mem = MergeMemNode::make(mem);
+ transform_later(local_mem);
+
+ didit = generate_block_arraycopy(&local_ctrl, &local_mem, local_io,
+ adr_type, basic_elem_type, alloc,
+ src, src_offset, dest, dest_offset,
+ dest_size, dest_uninitialized);
+ if (didit) {
+ // Present the results of the block-copying fast call.
+ result_region->init_req(bcopy_path, local_ctrl);
+ result_i_o ->init_req(bcopy_path, local_io);
+ result_memory->init_req(bcopy_path, local_mem->memory_at(alias_idx));
+ }
+ }
+ if (didit) {
+ *ctrl = top(); // no regular fast path
+ }
+ }
+
+ // Clear the tail, if any.
+ if (tail_ctl != NULL) {
+ Node* notail_ctl = (*ctrl)->is_top() ? NULL : *ctrl;
+ *ctrl = tail_ctl;
+ if (notail_ctl == NULL) {
+ generate_clear_array(*ctrl, mem,
+ adr_type, dest, basic_elem_type,
+ dest_tail, NULL,
+ dest_size);
+ } else {
+ // Make a local merge.
+ Node* done_ctl = transform_later(new RegionNode(3));
+ Node* done_mem = transform_later(new PhiNode(done_ctl, Type::MEMORY, adr_type));
+ done_ctl->init_req(1, notail_ctl);
+ done_mem->init_req(1, mem->memory_at(alias_idx));
+ generate_clear_array(*ctrl, mem,
+ adr_type, dest, basic_elem_type,
+ dest_tail, NULL,
+ dest_size);
+ done_ctl->init_req(2, *ctrl);
+ done_mem->init_req(2, mem->memory_at(alias_idx));
+ *ctrl = done_ctl;
+ mem->set_memory_at(alias_idx, done_mem);
+ }
+ }
+ }
+
+ BasicType copy_type = basic_elem_type;
+ assert(basic_elem_type != T_ARRAY, "caller must fix this");
+ if (!(*ctrl)->is_top() && copy_type == T_OBJECT) {
+ // If src and dest have compatible element types, we can copy bits.
+ // Types S[] and D[] are compatible if D is a supertype of S.
+ //
+ // If they are not, we will use checked_oop_disjoint_arraycopy,
+ // which performs a fast optimistic per-oop check, and backs off
+ // further to JVM_ArrayCopy on the first per-oop check that fails.
+ // (Actually, we don't move raw bits only; the GC requires card marks.)
+
+ // Get the klass* for both src and dest
+ Node* src_klass = ac->in(ArrayCopyNode::SrcKlass);
+ Node* dest_klass = ac->in(ArrayCopyNode::DestKlass);
+
+ // Generate the subtype check.
+ // This might fold up statically, or then again it might not.
+ //
+ // Non-static example: Copying List<String>.elements to a new String[].
+ // The backing store for a List<String> is always an Object[],
+ // but its elements are always type String, if the generic types
+ // are correct at the source level.
+ //
+ // Test S[] against D[], not S against D, because (probably)
+ // the secondary supertype cache is less busy for S[] than S.
+ // This usually only matters when D is an interface.
+ Node* not_subtype_ctrl = ac->is_arraycopy_notest() ? top() : Phase::gen_subtype_check(src_klass, dest_klass, ctrl, mem, &_igvn);
+ // Plug failing path into checked_oop_disjoint_arraycopy
+ if (not_subtype_ctrl != top()) {
+ Node* local_ctrl = not_subtype_ctrl;
+ MergeMemNode* local_mem = MergeMemNode::make(mem);
+ transform_later(local_mem);
+
+ // (At this point we can assume disjoint_bases, since types differ.)
+ int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
+ Node* p1 = basic_plus_adr(dest_klass, ek_offset);
+ Node* n1 = LoadKlassNode::make(_igvn, C->immutable_memory(), p1, TypeRawPtr::BOTTOM);
+ Node* dest_elem_klass = transform_later(n1);
+ Node* cv = generate_checkcast_arraycopy(&local_ctrl, &local_mem,
+ adr_type,
+ dest_elem_klass,
+ src, src_offset, dest, dest_offset,
+ ConvI2X(copy_length), dest_uninitialized);
+ if (cv == NULL) cv = intcon(-1); // failure (no stub available)
+ checked_control = local_ctrl;
+ checked_i_o = *io;
+ checked_mem = local_mem->memory_at(alias_idx);
+ checked_value = cv;
+ }
+ // At this point we know we do not need type checks on oop stores.
+
+ // Let's see if we need card marks:
+ if (alloc != NULL && GraphKit::use_ReduceInitialCardMarks()) {
+ // If we do not need card marks, copy using the jint or jlong stub.
+ copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
+ assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
+ "sizes agree");
+ }
+ }
+
+ if (!(*ctrl)->is_top()) {
+ // Generate the fast path, if possible.
+ Node* local_ctrl = *ctrl;
+ MergeMemNode* local_mem = MergeMemNode::make(mem);
+ transform_later(local_mem);
+
+ generate_unchecked_arraycopy(&local_ctrl, &local_mem,
+ adr_type, copy_type, disjoint_bases,
+ src, src_offset, dest, dest_offset,
+ ConvI2X(copy_length), dest_uninitialized);
+
+ // Present the results of the fast call.
+ result_region->init_req(fast_path, local_ctrl);
+ result_i_o ->init_req(fast_path, *io);
+ result_memory->init_req(fast_path, local_mem->memory_at(alias_idx));
+ }
+
+ // Here are all the slow paths up to this point, in one bundle:
+ assert(slow_region != NULL, "allocated on entry");
+ slow_control = slow_region;
+ DEBUG_ONLY(slow_region = (RegionNode*)badAddress);
+
+ *ctrl = checked_control;
+ if (!(*ctrl)->is_top()) {
+ // Clean up after the checked call.
+ // The returned value is either 0 or -1^K,
+ // where K = number of partially transferred array elements.
+ Node* cmp = new CmpINode(checked_value, intcon(0));
+ transform_later(cmp);
+ Node* bol = new BoolNode(cmp, BoolTest::eq);
+ transform_later(bol);
+ IfNode* iff = new IfNode(*ctrl, bol, PROB_MAX, COUNT_UNKNOWN);
+ transform_later(iff);
+
+ // If it is 0, we are done, so transfer to the end.
+ Node* checks_done = new IfTrueNode(iff);
+ transform_later(checks_done);
+ result_region->init_req(checked_path, checks_done);
+ result_i_o ->init_req(checked_path, checked_i_o);
+ result_memory->init_req(checked_path, checked_mem);
+
+ // If it is not zero, merge into the slow call.
+ *ctrl = new IfFalseNode(iff);
+ transform_later(*ctrl);
+ RegionNode* slow_reg2 = new RegionNode(3);
+ PhiNode* slow_i_o2 = new PhiNode(slow_reg2, Type::ABIO);
+ PhiNode* slow_mem2 = new PhiNode(slow_reg2, Type::MEMORY, adr_type);
+ transform_later(slow_reg2);
+ transform_later(slow_i_o2);
+ transform_later(slow_mem2);
+ slow_reg2 ->init_req(1, slow_control);
+ slow_i_o2 ->init_req(1, slow_i_o);
+ slow_mem2 ->init_req(1, slow_mem);
+ slow_reg2 ->init_req(2, *ctrl);
+ slow_i_o2 ->init_req(2, checked_i_o);
+ slow_mem2 ->init_req(2, checked_mem);
+
+ slow_control = slow_reg2;
+ slow_i_o = slow_i_o2;
+ slow_mem = slow_mem2;
+
+ if (alloc != NULL) {
+ // We'll restart from the very beginning, after zeroing the whole thing.
+ // This can cause double writes, but that's OK since dest is brand new.
+ // So we ignore the low 31 bits of the value returned from the stub.
+ } else {
+ // We must continue the copy exactly where it failed, or else
+ // another thread might see the wrong number of writes to dest.
+ Node* checked_offset = new XorINode(checked_value, intcon(-1));
+ Node* slow_offset = new PhiNode(slow_reg2, TypeInt::INT);
+ transform_later(checked_offset);
+ transform_later(slow_offset);
+ slow_offset->init_req(1, intcon(0));
+ slow_offset->init_req(2, checked_offset);
+
+ // Adjust the arguments by the conditionally incoming offset.
+ Node* src_off_plus = new AddINode(src_offset, slow_offset);
+ transform_later(src_off_plus);
+ Node* dest_off_plus = new AddINode(dest_offset, slow_offset);
+ transform_later(dest_off_plus);
+ Node* length_minus = new SubINode(copy_length, slow_offset);
+ transform_later(length_minus);
+
+ // Tweak the node variables to adjust the code produced below:
+ src_offset = src_off_plus;
+ dest_offset = dest_off_plus;
+ copy_length = length_minus;
+ }
+ }
+ *ctrl = slow_control;
+ if (!(*ctrl)->is_top()) {
+ Node* local_ctrl = *ctrl, *local_io = slow_i_o;
+ MergeMemNode* local_mem = MergeMemNode::make(mem);
+ transform_later(local_mem);
+
+ // Generate the slow path, if needed.
+ local_mem->set_memory_at(alias_idx, slow_mem);
+
+ if (dest_uninitialized) {
+ generate_clear_array(local_ctrl, local_mem,
+ adr_type, dest, basic_elem_type,
+ intcon(0), NULL,
+ alloc->in(AllocateNode::AllocSize));
+ }
+
+ local_mem = generate_slow_arraycopy(ac,
+ &local_ctrl, local_mem, &local_io,
+ adr_type,
+ src, src_offset, dest, dest_offset,
+ copy_length, /*dest_uninitialized*/false);
+
+ result_region->init_req(slow_call_path, local_ctrl);
+ result_i_o ->init_req(slow_call_path, local_io);
+ result_memory->init_req(slow_call_path, local_mem->memory_at(alias_idx));
+ } else {
+ ShouldNotReachHere(); // no call to generate_slow_arraycopy:
+ // projections were not extracted
+ }
+
+ // Remove unused edges.
+ for (uint i = 1; i < result_region->req(); i++) {
+ if (result_region->in(i) == NULL) {
+ result_region->init_req(i, top());
+ }
+ }
+
+ // Finished; return the combined state.
+ *ctrl = result_region;
+ *io = result_i_o;
+ mem->set_memory_at(alias_idx, result_memory);
+
+ // mem no longer guaranteed to stay a MergeMemNode
+ Node* out_mem = mem;
+ DEBUG_ONLY(mem = NULL);
+
+ // The memory edges above are precise in order to model effects around
+ // array copies accurately to allow value numbering of field loads around
+ // arraycopy. Such field loads, both before and after, are common in Java
+ // collections and similar classes involving header/array data structures.
+ //
+ // But with low number of register or when some registers are used or killed
+ // by arraycopy calls it causes registers spilling on stack. See 6544710.
+ // The next memory barrier is added to avoid it. If the arraycopy can be
+ // optimized away (which it can, sometimes) then we can manually remove
+ // the membar also.
+ //
+ // Do not let reads from the cloned object float above the arraycopy.
+ if (alloc != NULL && !alloc->initialization()->does_not_escape()) {
+ // Do not let stores that initialize this object be reordered with
+ // a subsequent store that would make this object accessible by
+ // other threads.
+ insert_mem_bar(ctrl, &out_mem, Op_MemBarStoreStore);
+ } else if (InsertMemBarAfterArraycopy) {
+ insert_mem_bar(ctrl, &out_mem, Op_MemBarCPUOrder);
+ }
+
+ _igvn.replace_node(_memproj_fallthrough, out_mem);
+ _igvn.replace_node(_ioproj_fallthrough, *io);
+ _igvn.replace_node(_fallthroughcatchproj, *ctrl);
+
+ return out_mem;
+}
+
+// Helper for initialization of arrays, creating a ClearArray.
+// It writes zero bits in [start..end), within the body of an array object.
+// The memory effects are all chained onto the 'adr_type' alias category.
+//
+// Since the object is otherwise uninitialized, we are free
+// to put a little "slop" around the edges of the cleared area,
+// as long as it does not go back into the array's header,
+// or beyond the array end within the heap.
+//
+// The lower edge can be rounded down to the nearest jint and the
+// upper edge can be rounded up to the nearest MinObjAlignmentInBytes.
+//
+// Arguments:
+// adr_type memory slice where writes are generated
+// dest oop of the destination array
+// basic_elem_type element type of the destination
+// slice_idx array index of first element to store
+// slice_len number of elements to store (or NULL)
+// dest_size total size in bytes of the array object
+//
+// Exactly one of slice_len or dest_size must be non-NULL.
+// If dest_size is non-NULL, zeroing extends to the end of the object.
+// If slice_len is non-NULL, the slice_idx value must be a constant.
+void PhaseMacroExpand::generate_clear_array(Node* ctrl, MergeMemNode* merge_mem,
+ const TypePtr* adr_type,
+ Node* dest,
+ BasicType basic_elem_type,
+ Node* slice_idx,
+ Node* slice_len,
+ Node* dest_size) {
+ // one or the other but not both of slice_len and dest_size:
+ assert((slice_len != NULL? 1: 0) + (dest_size != NULL? 1: 0) == 1, "");
+ if (slice_len == NULL) slice_len = top();
+ if (dest_size == NULL) dest_size = top();
+
+ uint alias_idx = C->get_alias_index(adr_type);
+
+ // operate on this memory slice:
+ Node* mem = merge_mem->memory_at(alias_idx); // memory slice to operate on
+
+ // scaling and rounding of indexes:
+ int scale = exact_log2(type2aelembytes(basic_elem_type));
+ int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
+ int clear_low = (-1 << scale) & (BytesPerInt - 1);
+ int bump_bit = (-1 << scale) & BytesPerInt;
+
+ // determine constant starts and ends
+ const intptr_t BIG_NEG = -128;
+ assert(BIG_NEG + 2*abase < 0, "neg enough");
+ intptr_t slice_idx_con = (intptr_t) _igvn.find_int_con(slice_idx, BIG_NEG);
+ intptr_t slice_len_con = (intptr_t) _igvn.find_int_con(slice_len, BIG_NEG);
+ if (slice_len_con == 0) {
+ return; // nothing to do here
+ }
+ intptr_t start_con = (abase + (slice_idx_con << scale)) & ~clear_low;
+ intptr_t end_con = _igvn.find_intptr_t_con(dest_size, -1);
+ if (slice_idx_con >= 0 && slice_len_con >= 0) {
+ assert(end_con < 0, "not two cons");
+ end_con = round_to(abase + ((slice_idx_con + slice_len_con) << scale),
+ BytesPerLong);
+ }
+
+ if (start_con >= 0 && end_con >= 0) {
+ // Constant start and end. Simple.
+ mem = ClearArrayNode::clear_memory(ctrl, mem, dest,
+ start_con, end_con, &_igvn);
+ } else if (start_con >= 0 && dest_size != top()) {
+ // Constant start, pre-rounded end after the tail of the array.
+ Node* end = dest_size;
+ mem = ClearArrayNode::clear_memory(ctrl, mem, dest,
+ start_con, end, &_igvn);
+ } else if (start_con >= 0 && slice_len != top()) {
+ // Constant start, non-constant end. End needs rounding up.
+ // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8)
+ intptr_t end_base = abase + (slice_idx_con << scale);
+ int end_round = (-1 << scale) & (BytesPerLong - 1);
+ Node* end = ConvI2X(slice_len);
+ if (scale != 0)
+ end = transform_later(new LShiftXNode(end, intcon(scale) ));
+ end_base += end_round;
+ end = transform_later(new AddXNode(end, MakeConX(end_base)) );
+ end = transform_later(new AndXNode(end, MakeConX(~end_round)) );
+ mem = ClearArrayNode::clear_memory(ctrl, mem, dest,
+ start_con, end, &_igvn);
+ } else if (start_con < 0 && dest_size != top()) {
+ // Non-constant start, pre-rounded end after the tail of the array.
+ // This is almost certainly a "round-to-end" operation.
+ Node* start = slice_idx;
+ start = ConvI2X(start);
+ if (scale != 0)
+ start = transform_later(new LShiftXNode( start, intcon(scale) ));
+ start = transform_later(new AddXNode(start, MakeConX(abase)) );
+ if ((bump_bit | clear_low) != 0) {
+ int to_clear = (bump_bit | clear_low);
+ // Align up mod 8, then store a jint zero unconditionally
+ // just before the mod-8 boundary.
+ if (((abase + bump_bit) & ~to_clear) - bump_bit
+ < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) {
+ bump_bit = 0;
+ assert((abase & to_clear) == 0, "array base must be long-aligned");
+ } else {
+ // Bump 'start' up to (or past) the next jint boundary:
+ start = transform_later( new AddXNode(start, MakeConX(bump_bit)) );
+ assert((abase & clear_low) == 0, "array base must be int-aligned");
+ }
+ // Round bumped 'start' down to jlong boundary in body of array.
+ start = transform_later(new AndXNode(start, MakeConX(~to_clear)) );
+ if (bump_bit != 0) {
+ // Store a zero to the immediately preceding jint:
+ Node* x1 = transform_later(new AddXNode(start, MakeConX(-bump_bit)) );
+ Node* p1 = basic_plus_adr(dest, x1);
+ mem = StoreNode::make(_igvn, ctrl, mem, p1, adr_type, intcon(0), T_INT, MemNode::unordered);
+ mem = transform_later(mem);
+ }
+ }
+ Node* end = dest_size; // pre-rounded
+ mem = ClearArrayNode::clear_memory(ctrl, mem, dest,
+ start, end, &_igvn);
+ } else {
+ // Non-constant start, unrounded non-constant end.
+ // (Nobody zeroes a random midsection of an array using this routine.)
+ ShouldNotReachHere(); // fix caller
+ }
+
+ // Done.
+ merge_mem->set_memory_at(alias_idx, mem);
+}
+
+bool PhaseMacroExpand::generate_block_arraycopy(Node** ctrl, MergeMemNode** mem, Node* io,
+ const TypePtr* adr_type,
+ BasicType basic_elem_type,
+ AllocateNode* alloc,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* dest_size, bool dest_uninitialized) {
+ // See if there is an advantage from block transfer.
+ int scale = exact_log2(type2aelembytes(basic_elem_type));
+ if (scale >= LogBytesPerLong)
+ return false; // it is already a block transfer
+
+ // Look at the alignment of the starting offsets.
+ int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
+
+ intptr_t src_off_con = (intptr_t) _igvn.find_int_con(src_offset, -1);
+ intptr_t dest_off_con = (intptr_t) _igvn.find_int_con(dest_offset, -1);
+ if (src_off_con < 0 || dest_off_con < 0) {
+ // At present, we can only understand constants.
+ return false;
+ }
+
+ intptr_t src_off = abase + (src_off_con << scale);
+ intptr_t dest_off = abase + (dest_off_con << scale);
+
+ if (((src_off | dest_off) & (BytesPerLong-1)) != 0) {
+ // Non-aligned; too bad.
+ // One more chance: Pick off an initial 32-bit word.
+ // This is a common case, since abase can be odd mod 8.
+ if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt &&
+ ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) {
+ Node* sptr = basic_plus_adr(src, src_off);
+ Node* dptr = basic_plus_adr(dest, dest_off);
+ uint alias_idx = C->get_alias_index(adr_type);
+ Node* sval = transform_later(LoadNode::make(_igvn, *ctrl, (*mem)->memory_at(alias_idx), sptr, adr_type, TypeInt::INT, T_INT, MemNode::unordered));
+ Node* st = transform_later(StoreNode::make(_igvn, *ctrl, (*mem)->memory_at(alias_idx), dptr, adr_type, sval, T_INT, MemNode::unordered));
+ (*mem)->set_memory_at(alias_idx, st);
+ src_off += BytesPerInt;
+ dest_off += BytesPerInt;
+ } else {
+ return false;
+ }
+ }
+ assert(src_off % BytesPerLong == 0, "");
+ assert(dest_off % BytesPerLong == 0, "");
+
+ // Do this copy by giant steps.
+ Node* sptr = basic_plus_adr(src, src_off);
+ Node* dptr = basic_plus_adr(dest, dest_off);
+ Node* countx = dest_size;
+ countx = transform_later(new SubXNode(countx, MakeConX(dest_off)));
+ countx = transform_later(new URShiftXNode(countx, intcon(LogBytesPerLong)));
+
+ bool disjoint_bases = true; // since alloc != NULL
+ generate_unchecked_arraycopy(ctrl, mem,
+ adr_type, T_LONG, disjoint_bases,
+ sptr, NULL, dptr, NULL, countx, dest_uninitialized);
+
+ return true;
+}
+
+// Helper function; generates code for the slow case.
+// We make a call to a runtime method which emulates the native method,
+// but without the native wrapper overhead.
+MergeMemNode* PhaseMacroExpand::generate_slow_arraycopy(ArrayCopyNode *ac,
+ Node** ctrl, Node* mem, Node** io,
+ const TypePtr* adr_type,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length, bool dest_uninitialized) {
+ assert(!dest_uninitialized, "Invariant");
+
+ const TypeFunc* call_type = OptoRuntime::slow_arraycopy_Type();
+ CallNode* call = new CallStaticJavaNode(call_type, OptoRuntime::slow_arraycopy_Java(),
+ "slow_arraycopy",
+ ac->jvms()->bci(), TypePtr::BOTTOM);
+
+ call->init_req(TypeFunc::Control, *ctrl);
+ call->init_req(TypeFunc::I_O , *io);
+ call->init_req(TypeFunc::Memory , mem);
+ call->init_req(TypeFunc::ReturnAdr, top());
+ call->init_req(TypeFunc::FramePtr, top());
+ call->init_req(TypeFunc::Parms+0, src);
+ call->init_req(TypeFunc::Parms+1, src_offset);
+ call->init_req(TypeFunc::Parms+2, dest);
+ call->init_req(TypeFunc::Parms+3, dest_offset);
+ call->init_req(TypeFunc::Parms+4, copy_length);
+ copy_call_debug_info(ac, call);
+
+ call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON.
+ _igvn.replace_node(ac, call);
+ transform_later(call);
+
+ extract_call_projections(call);
+ *ctrl = _fallthroughcatchproj->clone();
+ transform_later(*ctrl);
+
+ Node* m = _memproj_fallthrough->clone();
+ transform_later(m);
+
+ uint alias_idx = C->get_alias_index(adr_type);
+ MergeMemNode* out_mem;
+ if (alias_idx != Compile::AliasIdxBot) {
+ out_mem = MergeMemNode::make(mem);
+ out_mem->set_memory_at(alias_idx, m);
+ } else {
+ out_mem = MergeMemNode::make(m);
+ }
+ transform_later(out_mem);
+
+ *io = _ioproj_fallthrough->clone();
+ transform_later(*io);
+
+ return out_mem;
+}
+
+// Helper function; generates code for cases requiring runtime checks.
+Node* PhaseMacroExpand::generate_checkcast_arraycopy(Node** ctrl, MergeMemNode** mem,
+ const TypePtr* adr_type,
+ Node* dest_elem_klass,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length, bool dest_uninitialized) {
+ if ((*ctrl)->is_top()) return NULL;
+
+ address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized);
+ if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
+ return NULL;
+ }
+
+ // Pick out the parameters required to perform a store-check
+ // for the target array. This is an optimistic check. It will
+ // look in each non-null element's class, at the desired klass's
+ // super_check_offset, for the desired klass.
+ int sco_offset = in_bytes(Klass::super_check_offset_offset());
+ Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset);
+ Node* n3 = new LoadINode(NULL, *mem /*memory(p3)*/, p3, _igvn.type(p3)->is_ptr(), TypeInt::INT, MemNode::unordered);
+ Node* check_offset = ConvI2X(transform_later(n3));
+ Node* check_value = dest_elem_klass;
+
+ Node* src_start = array_element_address(src, src_offset, T_OBJECT);
+ Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT);
+
+ const TypeFunc* call_type = OptoRuntime::checkcast_arraycopy_Type();
+ Node* call = make_leaf_call(*ctrl, *mem, call_type, copyfunc_addr, "checkcast_arraycopy", adr_type,
+ src_start, dest_start, copy_length XTOP, check_offset XTOP, check_value);
+
+ finish_arraycopy_call(call, ctrl, mem, adr_type);
+
+ Node* proj = new ProjNode(call, TypeFunc::Parms);
+ transform_later(proj);
+
+ return proj;
+}
+
+// Helper function; generates code for cases requiring runtime checks.
+Node* PhaseMacroExpand::generate_generic_arraycopy(Node** ctrl, MergeMemNode** mem,
+ const TypePtr* adr_type,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length, bool dest_uninitialized) {
+ if ((*ctrl)->is_top()) return NULL;
+ assert(!dest_uninitialized, "Invariant");
+
+ address copyfunc_addr = StubRoutines::generic_arraycopy();
+ if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
+ return NULL;
+ }
+
+ const TypeFunc* call_type = OptoRuntime::generic_arraycopy_Type();
+ Node* call = make_leaf_call(*ctrl, *mem, call_type, copyfunc_addr, "generic_arraycopy", adr_type,
+ src, src_offset, dest, dest_offset, copy_length);
+
+ finish_arraycopy_call(call, ctrl, mem, adr_type);
+
+ Node* proj = new ProjNode(call, TypeFunc::Parms);
+ transform_later(proj);
+
+ return proj;
+}
+
+// Helper function; generates the fast out-of-line call to an arraycopy stub.
+void PhaseMacroExpand::generate_unchecked_arraycopy(Node** ctrl, MergeMemNode** mem,
+ const TypePtr* adr_type,
+ BasicType basic_elem_type,
+ bool disjoint_bases,
+ Node* src, Node* src_offset,
+ Node* dest, Node* dest_offset,
+ Node* copy_length, bool dest_uninitialized) {
+ if ((*ctrl)->is_top()) return;
+
+ Node* src_start = src;
+ Node* dest_start = dest;
+ if (src_offset != NULL || dest_offset != NULL) {
+ src_start = array_element_address(src, src_offset, basic_elem_type);
+ dest_start = array_element_address(dest, dest_offset, basic_elem_type);
+ }
+
+ // Figure out which arraycopy runtime method to call.
+ const char* copyfunc_name = "arraycopy";
+ address copyfunc_addr =
+ basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
+ disjoint_bases, copyfunc_name, dest_uninitialized);
+
+ const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
+ Node* call = make_leaf_call(*ctrl, *mem, call_type, copyfunc_addr, copyfunc_name, adr_type,
+ src_start, dest_start, copy_length XTOP);
+
+ finish_arraycopy_call(call, ctrl, mem, adr_type);
+}
+
+void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) {
+ Node* ctrl = ac->in(TypeFunc::Control);
+ Node* io = ac->in(TypeFunc::I_O);
+ Node* src = ac->in(ArrayCopyNode::Src);
+ Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
+ Node* dest = ac->in(ArrayCopyNode::Dest);
+ Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
+ Node* length = ac->in(ArrayCopyNode::Length);
+ MergeMemNode* merge_mem = NULL;
+
+ if (ac->is_clonebasic()) {
+ assert (src_offset == NULL && dest_offset == NULL, "for clone offsets should be null");
+ Node* mem = ac->in(TypeFunc::Memory);
+ const char* copyfunc_name = "arraycopy";
+ address copyfunc_addr =
+ basictype2arraycopy(T_LONG, NULL, NULL,
+ true, copyfunc_name, true);
+
+ const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
+ const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
+
+ Node* call = make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, src, dest, length XTOP);
+ transform_later(call);
+
+ _igvn.replace_node(ac, call);
+ return;
+ } else if (ac->is_copyof() || ac->is_copyofrange() || ac->is_cloneoop()) {
+ Node* mem = ac->in(TypeFunc::Memory);
+ merge_mem = MergeMemNode::make(mem);
+ transform_later(merge_mem);
+
+ RegionNode* slow_region = new RegionNode(1);
+ transform_later(slow_region);
+
+ AllocateArrayNode* alloc = NULL;
+ if (ac->is_alloc_tightly_coupled()) {
+ alloc = AllocateArrayNode::Ideal_array_allocation(dest, &_igvn);
+ assert(alloc != NULL, "expect alloc");
+ }
+
+ generate_arraycopy(ac, alloc, &ctrl, merge_mem, &io,
+ TypeAryPtr::OOPS, T_OBJECT,
+ src, src_offset, dest, dest_offset, length,
+ true, !ac->is_copyofrange());
+
+ return;
+ }
+
+ AllocateArrayNode* alloc = NULL;
+ if (ac->is_alloc_tightly_coupled()) {
+ alloc = AllocateArrayNode::Ideal_array_allocation(dest, &_igvn);
+ assert(alloc != NULL, "expect alloc");
+ }
+
+ assert(ac->is_arraycopy() || ac->is_arraycopy_notest(), "should be an arraycopy");
+
+ // Compile time checks. If any of these checks cannot be verified at compile time,
+ // we do not make a fast path for this call. Instead, we let the call remain as it
+ // is. The checks we choose to mandate at compile time are:
+ //
+ // (1) src and dest are arrays.
+ const Type* src_type = src->Value(&_igvn);
+ const Type* dest_type = dest->Value(&_igvn);
+ const TypeAryPtr* top_src = src_type->isa_aryptr();
+ const TypeAryPtr* top_dest = dest_type->isa_aryptr();
+
+ if (top_src == NULL || top_src->klass() == NULL ||
+ top_dest == NULL || top_dest->klass() == NULL) {
+ // Conservatively insert a memory barrier on all memory slices.
+ // Do not let writes into the source float below the arraycopy.
+ {
+ Node* mem = ac->in(TypeFunc::Memory);
+ insert_mem_bar(&ctrl, &mem, Op_MemBarCPUOrder);
+
+ merge_mem = MergeMemNode::make(mem);
+ transform_later(merge_mem);
+ }
+
+ // Call StubRoutines::generic_arraycopy stub.
+ Node* mem = generate_arraycopy(ac, NULL, &ctrl, merge_mem, &io,
+ TypeRawPtr::BOTTOM, T_CONFLICT,
+ src, src_offset, dest, dest_offset, length);
+
+ // Do not let reads from the destination float above the arraycopy.
+ // Since we cannot type the arrays, we don't know which slices
+ // might be affected. We could restrict this barrier only to those
+ // memory slices which pertain to array elements--but don't bother.
+ if (!InsertMemBarAfterArraycopy) {
+ // (If InsertMemBarAfterArraycopy, there is already one in place.)
+ insert_mem_bar(&ctrl, &mem, Op_MemBarCPUOrder);
+ }
+ return;
+ }
+ // (2) src and dest arrays must have elements of the same BasicType
+ // Figure out the size and type of the elements we will be copying.
+ BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type();
+ BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
+ if (src_elem == T_ARRAY) src_elem = T_OBJECT;
+ if (dest_elem == T_ARRAY) dest_elem = T_OBJECT;
+
+ if (src_elem != dest_elem || dest_elem == T_VOID) {
+ // The component types are not the same or are not recognized. Punt.
+ // (But, avoid the native method wrapper to JVM_ArrayCopy.)
+ {
+ Node* mem = ac->in(TypeFunc::Memory);
+ merge_mem = generate_slow_arraycopy(ac, &ctrl, mem, &io, TypePtr::BOTTOM, src, src_offset, dest, dest_offset, length, false);
+ }
+
+ _igvn.replace_node(_memproj_fallthrough, merge_mem);
+ _igvn.replace_node(_ioproj_fallthrough, io);
+ _igvn.replace_node(_fallthroughcatchproj, ctrl);
+ return;
+ }
+
+ //---------------------------------------------------------------------------
+ // We will make a fast path for this call to arraycopy.
+
+ // We have the following tests left to perform:
+ //
+ // (3) src and dest must not be null.
+ // (4) src_offset must not be negative.
+ // (5) dest_offset must not be negative.
+ // (6) length must not be negative.
+ // (7) src_offset + length must not exceed length of src.
+ // (8) dest_offset + length must not exceed length of dest.
+ // (9) each element of an oop array must be assignable
+
+ {
+ Node* mem = ac->in(TypeFunc::Memory);
+ merge_mem = MergeMemNode::make(mem);
+ transform_later(merge_mem);
+ }
+
+ RegionNode* slow_region = new RegionNode(1);
+ transform_later(slow_region);
+
+ if (!ac->is_arraycopy_notest()) {
+ // (3) operands must not be null
+ // We currently perform our null checks with the null_check routine.
+ // This means that the null exceptions will be reported in the caller
+ // rather than (correctly) reported inside of the native arraycopy call.
+ // This should be corrected, given time. We do our null check with the
+ // stack pointer restored.
+ // null checks done library_call.cpp
+
+ // (4) src_offset must not be negative.
+ generate_negative_guard(&ctrl, src_offset, slow_region);
+
+ // (5) dest_offset must not be negative.
+ generate_negative_guard(&ctrl, dest_offset, slow_region);
+
+ // (6) length must not be negative (moved to generate_arraycopy()).
+ // generate_negative_guard(length, slow_region);
+
+ // (7) src_offset + length must not exceed length of src.
+ Node* alen = ac->in(ArrayCopyNode::SrcLen);
+ generate_limit_guard(&ctrl,
+ src_offset, length,
+ alen,
+ slow_region);
+
+ // (8) dest_offset + length must not exceed length of dest.
+ alen = ac->in(ArrayCopyNode::DestLen);
+ generate_limit_guard(&ctrl,
+ dest_offset, length,
+ alen,
+ slow_region);
+
+ // (9) each element of an oop array must be assignable
+ // The generate_arraycopy subroutine checks this.
+ }
+ // This is where the memory effects are placed:
+ const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem);
+ generate_arraycopy(ac, alloc, &ctrl, merge_mem, &io,
+ adr_type, dest_elem,
+ src, src_offset, dest, dest_offset, length,
+ false, false, slow_region);
+}
--- a/hotspot/src/share/vm/opto/node.cpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/opto/node.cpp Thu Aug 21 19:06:21 2014 +0000
@@ -1083,6 +1083,9 @@
if( this->is_Store() ) {
// Condition for back-to-back stores folding.
return n->Opcode() == op && n->in(MemNode::Memory) == this;
+ } else if (this->is_Load()) {
+ // Condition for removing an unused LoadNode from the MemBarAcquire precedence input
+ return n->Opcode() == Op_MemBarAcquire;
} else if( op == Op_AddL ) {
// Condition for convL2I(addL(x,y)) ==> addI(convL2I(x),convL2I(y))
return n->Opcode() == Op_ConvL2I && n->in(1) == this;
--- a/hotspot/src/share/vm/opto/node.hpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/opto/node.hpp Thu Aug 21 19:06:21 2014 +0000
@@ -40,6 +40,7 @@
class AliasInfo;
class AllocateArrayNode;
class AllocateNode;
+class ArrayCopyNode;
class Block;
class BoolNode;
class BoxLockNode;
@@ -561,6 +562,7 @@
DEFINE_CLASS_ID(AbstractLock, Call, 3)
DEFINE_CLASS_ID(Lock, AbstractLock, 0)
DEFINE_CLASS_ID(Unlock, AbstractLock, 1)
+ DEFINE_CLASS_ID(ArrayCopy, Call, 4)
DEFINE_CLASS_ID(MultiBranch, Multi, 1)
DEFINE_CLASS_ID(PCTable, MultiBranch, 0)
DEFINE_CLASS_ID(Catch, PCTable, 0)
@@ -707,6 +709,7 @@
DEFINE_CLASS_QUERY(AddP)
DEFINE_CLASS_QUERY(Allocate)
DEFINE_CLASS_QUERY(AllocateArray)
+ DEFINE_CLASS_QUERY(ArrayCopy)
DEFINE_CLASS_QUERY(Bool)
DEFINE_CLASS_QUERY(BoxLock)
DEFINE_CLASS_QUERY(Call)
--- a/hotspot/src/share/vm/opto/phase.hpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/opto/phase.hpp Thu Aug 21 19:06:21 2014 +0000
@@ -27,7 +27,10 @@
#include "runtime/timer.hpp"
-class Compile;
+class IfNode;
+class MergeMemNode;
+class Node;
+class PhaseGVN;
//------------------------------Phase------------------------------------------
// Most optimizations are done in Phases. Creating a phase does any long
@@ -114,9 +117,20 @@
static elapsedTimer _t_instrSched;
static elapsedTimer _t_buildOopMaps;
#endif
+
+ // Generate a subtyping check. Takes as input the subtype and supertype.
+ // Returns 2 values: sets the default control() to the true path and
+ // returns the false path. Only reads from constant memory taken from the
+ // default memory; does not write anything. It also doesn't take in an
+ // Object; if you wish to check an Object you need to load the Object's
+ // class prior to coming here.
+ // Used in GraphKit and PhaseMacroExpand
+ static Node* gen_subtype_check(Node* subklass, Node* superklass, Node** ctrl, MergeMemNode* mem, PhaseGVN* gvn);
+
public:
Compile * C;
Phase( PhaseNumber pnum );
+
#ifndef PRODUCT
static void print_timers();
#endif
--- a/hotspot/src/share/vm/opto/phaseX.hpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/opto/phaseX.hpp Thu Aug 21 19:06:21 2014 +0000
@@ -390,6 +390,9 @@
// in a faster or cheaper fashion.
Node *transform( Node *n );
Node *transform_no_reclaim( Node *n );
+ virtual void record_for_igvn(Node *n) {
+ C->record_for_igvn(n);
+ }
void replace_with(PhaseGVN* gvn) {
_table.replace_with(&gvn->_table);
@@ -418,9 +421,6 @@
protected:
- // Idealize new Node 'n' with respect to its inputs and its value
- virtual Node *transform( Node *a_node );
-
// Warm up hash table, type table and initial worklist
void init_worklist( Node *a_root );
@@ -434,6 +434,10 @@
PhaseIterGVN( PhaseGVN *gvn ); // Used after Parser
PhaseIterGVN( PhaseIterGVN *igvn, const char *dummy ); // Used after +VerifyOpto
+ // Idealize new Node 'n' with respect to its inputs and its value
+ virtual Node *transform( Node *a_node );
+ virtual void record_for_igvn(Node *n) { }
+
virtual PhaseIterGVN *is_IterGVN() { return this; }
Unique_Node_List _worklist; // Iterative worklist
--- a/hotspot/src/share/vm/opto/subnode.cpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/opto/subnode.cpp Thu Aug 21 19:06:21 2014 +0000
@@ -1201,6 +1201,54 @@
return new BoolNode(in(1), _test.negate());
}
+// Change "bool eq/ne (cmp (add/sub A B) C)" into false/true if add/sub
+// overflows and we can prove that C is not in the two resulting ranges.
+// This optimization is similar to the one performed by CmpUNode::Value().
+Node* BoolNode::fold_cmpI(PhaseGVN* phase, SubNode* cmp, Node* cmp1, int cmp_op,
+ int cmp1_op, const TypeInt* cmp2_type) {
+ // Only optimize eq/ne integer comparison of add/sub
+ if((_test._test == BoolTest::eq || _test._test == BoolTest::ne) &&
+ (cmp_op == Op_CmpI) && (cmp1_op == Op_AddI || cmp1_op == Op_SubI)) {
+ // Skip cases were inputs of add/sub are not integers or of bottom type
+ const TypeInt* r0 = phase->type(cmp1->in(1))->isa_int();
+ const TypeInt* r1 = phase->type(cmp1->in(2))->isa_int();
+ if ((r0 != NULL) && (r0 != TypeInt::INT) &&
+ (r1 != NULL) && (r1 != TypeInt::INT) &&
+ (cmp2_type != TypeInt::INT)) {
+ // Compute exact (long) type range of add/sub result
+ jlong lo_long = r0->_lo;
+ jlong hi_long = r0->_hi;
+ if (cmp1_op == Op_AddI) {
+ lo_long += r1->_lo;
+ hi_long += r1->_hi;
+ } else {
+ lo_long -= r1->_hi;
+ hi_long -= r1->_lo;
+ }
+ // Check for over-/underflow by casting to integer
+ int lo_int = (int)lo_long;
+ int hi_int = (int)hi_long;
+ bool underflow = lo_long != (jlong)lo_int;
+ bool overflow = hi_long != (jlong)hi_int;
+ if ((underflow != overflow) && (hi_int < lo_int)) {
+ // Overflow on one boundary, compute resulting type ranges:
+ // tr1 [MIN_INT, hi_int] and tr2 [lo_int, MAX_INT]
+ int w = MAX2(r0->_widen, r1->_widen); // _widen does not matter here
+ const TypeInt* tr1 = TypeInt::make(min_jint, hi_int, w);
+ const TypeInt* tr2 = TypeInt::make(lo_int, max_jint, w);
+ // Compare second input of cmp to both type ranges
+ const Type* sub_tr1 = cmp->sub(tr1, cmp2_type);
+ const Type* sub_tr2 = cmp->sub(tr2, cmp2_type);
+ if (sub_tr1 == TypeInt::CC_LT && sub_tr2 == TypeInt::CC_GT) {
+ // The result of the add/sub will never equal cmp2. Replace BoolNode
+ // by false (0) if it tests for equality and by true (1) otherwise.
+ return ConINode::make((_test._test == BoolTest::eq) ? 0 : 1);
+ }
+ }
+ }
+ }
+ return NULL;
+}
//------------------------------Ideal------------------------------------------
Node *BoolNode::Ideal(PhaseGVN *phase, bool can_reshape) {
@@ -1294,6 +1342,9 @@
return new BoolNode( ncmp, _test.commute() );
}
+ // Try to optimize signed integer comparison
+ return fold_cmpI(phase, cmp->as_Sub(), cmp1, cop, cmp1_op, cmp2_type);
+
// The transformation below is not valid for either signed or unsigned
// comparisons due to wraparound concerns at MAX_VALUE and MIN_VALUE.
// This transformation can be resurrected when we are able to
@@ -1338,8 +1389,6 @@
// phase->type( cmp2->in(2) ) == TypeInt::ONE )
// return clone_cmp( cmp, cmp1, cmp2->in(1), phase, BoolTest::le );
// }
-
- return NULL;
}
//------------------------------Value------------------------------------------
--- a/hotspot/src/share/vm/opto/subnode.hpp Thu Aug 21 10:51:31 2014 -0700
+++ b/hotspot/src/share/vm/opto/subnode.hpp Thu Aug 21 19:06:21 2014 +0000
@@ -286,6 +286,10 @@
virtual uint hash() const;
virtual uint cmp( const Node &n ) const;
virtual uint size_of() const;
+
+ // Try to optimize signed integer comparison
+ Node* fold_cmpI(PhaseGVN* phase, SubNode* cmp, Node* cmp1, int cmp_op,
+ int cmp1_op, const TypeInt* cmp2_type);
public:
const BoolTest _test;
BoolNode( Node *cc, BoolTest::mask t): _test(t), Node(0,cc) {
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hotspot/test/compiler/IntegerArithmetic/TestIntegerComparison.java Thu Aug 21 19:06:21 2014 +0000
@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test TestIntegerComparison
+ * @bug 8043284 8042786
+ * @summary "Tests optimizations of signed and unsigned integer comparison."
+ * @run main/othervm -server -Xcomp -XX:+IgnoreUnrecognizedVMOptions -XX:-TieredCompilation -XX:CompileOnly=TestIntegerComparison::testSigned,TestIntegerComparison::testUnsigned TestIntegerComparison
+ */
+public class TestIntegerComparison {
+ /**
+ * Tests optimization of signed integer comparison (see BoolNode::Ideal).
+ * The body of the if statement is unreachable and should not be compiled.
+ * @param c Character (value in the integer range [0, 65535])
+ */
+ public static void testSigned(char c) {
+ // The following addition may overflow. The result is in one
+ // of the two ranges [IntMax] and [IntMin, IntMin + CharMax - 1].
+ int result = c + Integer.MAX_VALUE;
+ // CmpINode has to consider both result ranges instead of only
+ // the general [IntMin, IntMax] range to be able to prove that
+ // result is always unequal to CharMax.
+ if (result == Character.MAX_VALUE) {
+ // Unreachable
+ throw new RuntimeException("Should not reach here!");
+ }
+ }
+
+ /**
+ * Tests optimization of unsigned integer comparison (see CmpUNode::Value).
+ * The body of the if statement is unreachable and should not be compiled.
+ * @param c Character (value in the integer range [0, 65535])
+ */
+ public static void testUnsigned(char c) {
+ /*
+ * The following if statement consisting of two CmpIs is replaced
+ * by a CmpU during optimization (see 'IfNode::fold_compares').
+ *
+ * The signed (lo < i) and (i < hi) are replaced by the unsigned
+ * (i - (lo+1) < hi - (lo+1)). In this case the unsigned comparison
+ * equals (result - 2) < 98 leading to the following CmpUNode:
+ *
+ * CmpU (AddI result, -2) 98
+ *
+ * With the value of result this is simplified to:
+ *
+ * CmpU (AddI c, -(CharMax - IntMin)) 98
+ *
+ * The subtraction may underflow. The result is in one of the two
+ * ranges [IntMin], [IntMax - CharMax + 1]. Both ranges have to be
+ * considered instead of only the general [IntMin, IntMax] to prove
+ * that due to the overflow the signed comparison result < 98 is
+ * always false.
+ */
+ int result = c - (Character.MAX_VALUE - Integer.MIN_VALUE) + 2;
+ if (1 < result && result < 100) {
+ // Unreachable
+ throw new RuntimeException("Should not reach here!");
+ }
+ }
+
+ /**
+ * Tests optimizations of signed and unsigned integer comparison.
+ */
+ public static void main(String[] args) {
+ // We use characters to get a limited integer range for free
+ for (int i = Character.MIN_VALUE; i <= Character.MAX_VALUE; ++i) {
+ testSigned((char) i);
+ testUnsigned((char) i);
+ }
+ }
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hotspot/test/compiler/arraycopy/TestMissingControl.java Thu Aug 21 19:06:21 2014 +0000
@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8055153
+ * @summary missing control on LoadRange and LoadKlass when array copy macro node is expanded
+ * @run main/othervm -XX:-BackgroundCompilation -XX:-UseOnStackReplacement -XX:-TieredCompilation TestMissingControl
+ *
+ */
+public class TestMissingControl {
+
+ static int[] m1(int[] a2) {
+ int[] a1 = new int[10];
+ System.arraycopy(a1, 0, a2, 0, 10);
+ return a1;
+ }
+
+ static class A {
+ }
+
+ static Object m2(Object[] a2) {
+ A[] a1 = new A[10];
+ System.arraycopy(a1, 0, a2, 0, 10);
+ return a1;
+ }
+
+ static void test1() {
+ int[] a2 = new int[10];
+ int[] a3 = new int[5];
+
+ // compile m1 with arraycopy intrinsified
+ for (int i = 0; i < 20000; i++) {
+ m1(a2);
+ }
+
+ // make m1 trap
+ for (int i = 0; i < 10; i++) {
+ try {
+ m1(a3);
+ } catch(IndexOutOfBoundsException ioobe) {
+ }
+ }
+
+ // recompile m1
+ for (int i = 0; i < 20000; i++) {
+ m1(a2);
+ }
+
+ try {
+ m1(null);
+ } catch(NullPointerException npe) {}
+ }
+
+ static void test2() {
+ A[] a2 = new A[10];
+ A[] a3 = new A[5];
+
+ // compile m2 with arraycopy intrinsified
+ for (int i = 0; i < 20000; i++) {
+ m2(a2);
+ }
+
+ // make m2 trap
+ for (int i = 0; i < 10; i++) {
+ try {
+ m2(a3);
+ } catch(IndexOutOfBoundsException ioobe) {
+ }
+ }
+
+ // recompile m2
+ for (int i = 0; i < 20000; i++) {
+ m2(a2);
+ }
+
+ try {
+ m2(null);
+ } catch(NullPointerException npe) {}
+ }
+
+ static public void main(String[] args) {
+ test1();
+ test2();
+ }
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hotspot/test/compiler/classUnloading/anonymousClass/TestAnonymousClassUnloading.java Thu Aug 21 19:06:21 2014 +0000
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import sun.hotspot.WhiteBox;
+import sun.misc.Unsafe;
+import sun.misc.IOUtils;
+
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.net.URLConnection;
+
+/*
+ * @test TestAnonymousClassUnloading
+ * @bug 8054402
+ * @summary "Tests unloading of anonymous classes."
+ * @library /testlibrary /testlibrary/whitebox
+ * @compile TestAnonymousClassUnloading.java
+ * @run main ClassFileInstaller TestAnonymousClassUnloading sun.hotspot.WhiteBox
+ * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:-BackgroundCompilation TestAnonymousClassUnloading
+ */
+public class TestAnonymousClassUnloading {
+ private static final WhiteBox WHITE_BOX = WhiteBox.getWhiteBox();
+ private static final Unsafe UNSAFE = Unsafe.getUnsafe();
+ private static int COMP_LEVEL_SIMPLE = 1;
+ private static int COMP_LEVEL_FULL_OPTIMIZATION = 4;
+
+ /**
+ * We override hashCode here to be able to access this implementation
+ * via an Object reference (we cannot cast to TestAnonymousClassUnloading).
+ */
+ @Override
+ public int hashCode() {
+ return 42;
+ }
+
+ /**
+ * Does some work by using the anonymousClass.
+ * @param anonymousClass Class performing some work (will be unloaded)
+ */
+ static private void doWork(Class<?> anonymousClass) throws InstantiationException, IllegalAccessException {
+ // Create a new instance
+ Object anon = anonymousClass.newInstance();
+ // We would like to call a method of anonymousClass here but we cannot cast because the class
+ // was loaded by a different class loader. One solution would be to use reflection but since
+ // we want C2 to implement the call as an IC we call Object::hashCode() here which actually
+ // calls anonymousClass::hashCode(). C2 will then implement this call as an IC.
+ if (anon.hashCode() != 42) {
+ new RuntimeException("Work not done");
+ }
+ }
+
+ /**
+ * Makes sure that method is compiled by forcing compilation if not yet compiled.
+ * @param m Method to be checked
+ */
+ static private void makeSureIsCompiled(Method m) {
+ // Make sure background compilation is disabled
+ if (WHITE_BOX.getBooleanVMFlag("BackgroundCompilation")) {
+ throw new RuntimeException("Background compilation enabled");
+ }
+
+ // Check if already compiled
+ if (!WHITE_BOX.isMethodCompiled(m)) {
+ // If not, try to compile it with C2
+ if(!WHITE_BOX.enqueueMethodForCompilation(m, COMP_LEVEL_FULL_OPTIMIZATION)) {
+ // C2 compiler not available, try to compile with C1
+ WHITE_BOX.enqueueMethodForCompilation(m, COMP_LEVEL_SIMPLE);
+ }
+ // Because background compilation is disabled, method should now be compiled
+ if(!WHITE_BOX.isMethodCompiled(m)) {
+ throw new RuntimeException(m + " not compiled");
+ }
+ }
+ }
+
+ /**
+ * This test creates stale Klass* metadata referenced by a compiled IC.
+ *
+ * The following steps are performed:
+ * (1) An anonymous version of TestAnonymousClassUnloading is loaded by a custom class loader
+ * (2) The method doWork that calls a method of the anonymous class is compiled. The call
+ * is implemented as an IC referencing Klass* metadata of the anonymous class.
+ * (3) Unloading of the anonymous class is enforced. The IC now references dead metadata.
+ */
+ static public void main(String[] args) throws Exception {
+ // (1) Load an anonymous version of this class using the corresponding Unsafe method
+ URL classUrl = TestAnonymousClassUnloading.class.getResource("TestAnonymousClassUnloading.class");
+ URLConnection connection = classUrl.openConnection();
+ byte[] classBytes = IOUtils.readFully(connection.getInputStream(), connection.getContentLength(), true);
+ Class<?> anonymousClass = UNSAFE.defineAnonymousClass(TestAnonymousClassUnloading.class, classBytes, null);
+
+ // (2) Make sure all paths of doWork are profiled and compiled
+ for (int i = 0; i < 100000; ++i) {
+ doWork(anonymousClass);
+ }
+
+ // Make sure doWork is compiled now
+ Method doWork = TestAnonymousClassUnloading.class.getDeclaredMethod("doWork", Class.class);
+ makeSureIsCompiled(doWork);
+
+ // (3) Throw away reference to anonymousClass to allow unloading
+ anonymousClass = null;
+
+ // Force garbage collection to trigger unloading of anonymousClass
+ // Dead metadata reference to anonymousClass triggers JDK-8054402
+ WHITE_BOX.fullGC();
+ }
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hotspot/test/compiler/exceptions/TestRecursiveReplacedException.java Thu Aug 21 19:06:21 2014 +0000
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8054224
+ * @summary Recursive method compiled by C1 is unable to catch StackOverflowError
+ * @run main/othervm -Xcomp -XX:CompileOnly=Test.run -XX:+TieredCompilation -XX:TieredStopAtLevel=2 -Xss256K TestRecursiveReplacedException
+ *
+ */
+
+public class TestRecursiveReplacedException {
+
+ public static void main(String args[]) {
+ new TestRecursiveReplacedException().run();
+ }
+
+ public void run() {
+ try {
+ run();
+ } catch (Throwable t) {
+ }
+ }
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hotspot/test/compiler/membars/TestMemBarAcquire.java Thu Aug 21 19:06:21 2014 +0000
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test TestMemBarAcquire
+ * @bug 8048879
+ * @summary "Tests optimization of MemBarAcquireNodes"
+ * @run main/othervm -XX:-TieredCompilation -XX:-BackgroundCompilation TestMemBarAcquire
+ */
+public class TestMemBarAcquire {
+ private volatile static Object defaultObj = new Object();
+ private Object obj;
+
+ public TestMemBarAcquire(Object param) {
+ // Volatile load. MemBarAcquireNode is added after the
+ // load to prevent following loads from floating up past.
+ // StoreNode is added to store result of load in 'obj'.
+ this.obj = defaultObj;
+ // Overrides 'obj' and therefore makes previous StoreNode
+ // and the corresponding LoadNode useless. However, the
+ // LoadNode is still connected to the MemBarAcquireNode
+ // that should now release the reference.
+ this.obj = param;
+ }
+
+ public static void main(String[] args) throws Exception {
+ // Make sure TestMemBarAcquire::<init> is compiled
+ for (int i = 0; i < 100000; ++i) {
+ TestMemBarAcquire p = new TestMemBarAcquire(new Object());
+ }
+ }
+}
+