diff -r 50c604cb0d5f -r 30245956af37 hotspot/src/cpu/x86/vm/methodHandles_x86.cpp --- a/hotspot/src/cpu/x86/vm/methodHandles_x86.cpp Mon Jul 23 13:04:59 2012 -0700 +++ b/hotspot/src/cpu/x86/vm/methodHandles_x86.cpp Tue Jul 24 10:51:00 2012 -0700 @@ -32,8 +32,10 @@ #ifdef PRODUCT #define BLOCK_COMMENT(str) /* nothing */ +#define STOP(error) stop(error) #else #define BLOCK_COMMENT(str) __ block_comment(str) +#define STOP(error) block_comment(error); __ stop(error) #endif #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") @@ -43,483 +45,24 @@ return RegisterOrConstant(value); } -address MethodHandleEntry::start_compiled_entry(MacroAssembler* _masm, - address interpreted_entry) { - // Just before the actual machine code entry point, allocate space - // for a MethodHandleEntry::Data record, so that we can manage everything - // from one base pointer. - __ align(wordSize); - address target = __ pc() + sizeof(Data); - while (__ pc() < target) { - __ nop(); - __ align(wordSize); - } - - MethodHandleEntry* me = (MethodHandleEntry*) __ pc(); - me->set_end_address(__ pc()); // set a temporary end_address - me->set_from_interpreted_entry(interpreted_entry); - me->set_type_checking_entry(NULL); - - return (address) me; -} - -MethodHandleEntry* MethodHandleEntry::finish_compiled_entry(MacroAssembler* _masm, - address start_addr) { - MethodHandleEntry* me = (MethodHandleEntry*) start_addr; - assert(me->end_address() == start_addr, "valid ME"); - - // Fill in the real end_address: - __ align(wordSize); - me->set_end_address(__ pc()); - - return me; -} - -// stack walking support - -frame MethodHandles::ricochet_frame_sender(const frame& fr, RegisterMap *map) { - RicochetFrame* f = RicochetFrame::from_frame(fr); - if (map->update_map()) - frame::update_map_with_saved_link(map, &f->_sender_link); - return frame(f->extended_sender_sp(), f->exact_sender_sp(), f->sender_link(), f->sender_pc()); -} - -void MethodHandles::ricochet_frame_oops_do(const frame& fr, OopClosure* blk, const RegisterMap* reg_map) { - RicochetFrame* f = RicochetFrame::from_frame(fr); - - // pick up the argument type descriptor: - Thread* thread = Thread::current(); - Handle cookie(thread, f->compute_saved_args_layout(true, true)); - - // process fixed part - blk->do_oop((oop*)f->saved_target_addr()); - blk->do_oop((oop*)f->saved_args_layout_addr()); - - // process variable arguments: - if (cookie.is_null()) return; // no arguments to describe - - // the cookie is actually the invokeExact method for my target - // his argument signature is what I'm interested in - assert(cookie->is_method(), ""); - methodHandle invoker(thread, methodOop(cookie())); - assert(invoker->name() == vmSymbols::invokeExact_name(), "must be this kind of method"); - assert(!invoker->is_static(), "must have MH argument"); - int slot_count = invoker->size_of_parameters(); - assert(slot_count >= 1, "must include 'this'"); - intptr_t* base = f->saved_args_base(); - intptr_t* retval = NULL; - if (f->has_return_value_slot()) - retval = f->return_value_slot_addr(); - int slot_num = slot_count; - intptr_t* loc = &base[slot_num -= 1]; - //blk->do_oop((oop*) loc); // original target, which is irrelevant - int arg_num = 0; - for (SignatureStream ss(invoker->signature()); !ss.is_done(); ss.next()) { - if (ss.at_return_type()) continue; - BasicType ptype = ss.type(); - if (ptype == T_ARRAY) ptype = T_OBJECT; // fold all refs to T_OBJECT - assert(ptype >= T_BOOLEAN && ptype <= T_OBJECT, "not array or void"); - loc = &base[slot_num -= type2size[ptype]]; - bool is_oop = (ptype == T_OBJECT && loc != retval); - if (is_oop) blk->do_oop((oop*)loc); - arg_num += 1; - } - assert(slot_num == 0, "must have processed all the arguments"); -} - -oop MethodHandles::RicochetFrame::compute_saved_args_layout(bool read_cache, bool write_cache) { - oop cookie = NULL; - if (read_cache) { - cookie = saved_args_layout(); - if (cookie != NULL) return cookie; - } - oop target = saved_target(); - oop mtype = java_lang_invoke_MethodHandle::type(target); - oop mtform = java_lang_invoke_MethodType::form(mtype); - cookie = java_lang_invoke_MethodTypeForm::vmlayout(mtform); - if (write_cache) { - (*saved_args_layout_addr()) = cookie; - } - return cookie; -} - -void MethodHandles::RicochetFrame::generate_ricochet_blob(MacroAssembler* _masm, - // output params: - int* bounce_offset, - int* exception_offset, - int* frame_size_in_words) { - (*frame_size_in_words) = RicochetFrame::frame_size_in_bytes() / wordSize; - - address start = __ pc(); - -#ifdef ASSERT - __ hlt(); __ hlt(); __ hlt(); - // here's a hint of something special: - __ push(MAGIC_NUMBER_1); - __ push(MAGIC_NUMBER_2); -#endif //ASSERT - __ hlt(); // not reached - - // A return PC has just been popped from the stack. - // Return values are in registers. - // The ebp points into the RicochetFrame, which contains - // a cleanup continuation we must return to. - - (*bounce_offset) = __ pc() - start; - BLOCK_COMMENT("ricochet_blob.bounce"); - - if (VerifyMethodHandles) RicochetFrame::verify_clean(_masm); - trace_method_handle(_masm, "return/ricochet_blob.bounce"); - - __ jmp(frame_address(continuation_offset_in_bytes())); - __ hlt(); - DEBUG_ONLY(__ push(MAGIC_NUMBER_2)); - - (*exception_offset) = __ pc() - start; - BLOCK_COMMENT("ricochet_blob.exception"); - - // compare this to Interpreter::rethrow_exception_entry, which is parallel code - // for example, see TemplateInterpreterGenerator::generate_throw_exception - // Live registers in: - // rax: exception - // rdx: return address/pc that threw exception (ignored, always equal to bounce addr) - __ verify_oop(rax); - - // no need to empty_FPU_stack or reinit_heapbase, since caller frame will do the same if needed - - // Take down the frame. - - // Cf. InterpreterMacroAssembler::remove_activation. - leave_ricochet_frame(_masm, /*rcx_recv=*/ noreg, - saved_last_sp_register(), - /*sender_pc_reg=*/ rdx); - - // In between activations - previous activation type unknown yet - // compute continuation point - the continuation point expects the - // following registers set up: - // - // rax: exception - // rdx: return address/pc that threw exception - // rsp: expression stack of caller - // rbp: ebp of caller - __ push(rax); // save exception - __ push(rdx); // save return address - Register thread_reg = LP64_ONLY(r15_thread) NOT_LP64(rdi); - NOT_LP64(__ get_thread(thread_reg)); - __ call_VM_leaf(CAST_FROM_FN_PTR(address, - SharedRuntime::exception_handler_for_return_address), - thread_reg, rdx); - __ mov(rbx, rax); // save exception handler - __ pop(rdx); // restore return address - __ pop(rax); // restore exception - __ jmp(rbx); // jump to exception - // handler of caller -} - -void MethodHandles::RicochetFrame::enter_ricochet_frame(MacroAssembler* _masm, - Register rcx_recv, - Register rax_argv, - address return_handler, - Register rbx_temp) { - const Register saved_last_sp = saved_last_sp_register(); - Address rcx_mh_vmtarget( rcx_recv, java_lang_invoke_MethodHandle::vmtarget_offset_in_bytes() ); - Address rcx_amh_conversion( rcx_recv, java_lang_invoke_AdapterMethodHandle::conversion_offset_in_bytes() ); - - // Push the RicochetFrame a word at a time. - // This creates something similar to an interpreter frame. - // Cf. TemplateInterpreterGenerator::generate_fixed_frame. - BLOCK_COMMENT("push RicochetFrame {"); - DEBUG_ONLY(int rfo = (int) sizeof(RicochetFrame)); - assert((rfo -= wordSize) == RicochetFrame::sender_pc_offset_in_bytes(), ""); -#define RF_FIELD(push_value, name) \ - { push_value; \ - assert((rfo -= wordSize) == RicochetFrame::name##_offset_in_bytes(), ""); } - RF_FIELD(__ push(rbp), sender_link); - RF_FIELD(__ push(saved_last_sp), exact_sender_sp); // rsi/r13 - RF_FIELD(__ pushptr(rcx_amh_conversion), conversion); - RF_FIELD(__ push(rax_argv), saved_args_base); // can be updated if args are shifted - RF_FIELD(__ push((int32_t) NULL_WORD), saved_args_layout); // cache for GC layout cookie - if (UseCompressedOops) { - __ load_heap_oop(rbx_temp, rcx_mh_vmtarget); - RF_FIELD(__ push(rbx_temp), saved_target); - } else { - RF_FIELD(__ pushptr(rcx_mh_vmtarget), saved_target); - } - __ lea(rbx_temp, ExternalAddress(return_handler)); - RF_FIELD(__ push(rbx_temp), continuation); -#undef RF_FIELD - assert(rfo == 0, "fully initialized the RicochetFrame"); - // compute new frame pointer: - __ lea(rbp, Address(rsp, RicochetFrame::sender_link_offset_in_bytes())); - // Push guard word #1 in debug mode. - DEBUG_ONLY(__ push((int32_t) RicochetFrame::MAGIC_NUMBER_1)); - // For debugging, leave behind an indication of which stub built this frame. - DEBUG_ONLY({ Label L; __ call(L, relocInfo::none); __ bind(L); }); - BLOCK_COMMENT("} RicochetFrame"); -} - -void MethodHandles::RicochetFrame::leave_ricochet_frame(MacroAssembler* _masm, - Register rcx_recv, - Register new_sp_reg, - Register sender_pc_reg) { - assert_different_registers(rcx_recv, new_sp_reg, sender_pc_reg); - const Register saved_last_sp = saved_last_sp_register(); - // Take down the frame. - // Cf. InterpreterMacroAssembler::remove_activation. - BLOCK_COMMENT("end_ricochet_frame {"); - // TO DO: If (exact_sender_sp - extended_sender_sp) > THRESH, compact the frame down. - // This will keep stack in bounds even with unlimited tailcalls, each with an adapter. - if (rcx_recv->is_valid()) - __ movptr(rcx_recv, RicochetFrame::frame_address(RicochetFrame::saved_target_offset_in_bytes())); - __ movptr(sender_pc_reg, RicochetFrame::frame_address(RicochetFrame::sender_pc_offset_in_bytes())); - __ movptr(saved_last_sp, RicochetFrame::frame_address(RicochetFrame::exact_sender_sp_offset_in_bytes())); - __ movptr(rbp, RicochetFrame::frame_address(RicochetFrame::sender_link_offset_in_bytes())); - __ mov(rsp, new_sp_reg); - BLOCK_COMMENT("} end_ricochet_frame"); -} - -// Emit code to verify that RBP is pointing at a valid ricochet frame. -#ifndef PRODUCT -enum { - ARG_LIMIT = 255, SLOP = 4, - // use this parameter for checking for garbage stack movements: - UNREASONABLE_STACK_MOVE = (ARG_LIMIT + SLOP) - // the slop defends against false alarms due to fencepost errors -}; -#endif - -#ifdef ASSERT -void MethodHandles::RicochetFrame::verify_clean(MacroAssembler* _masm) { - // The stack should look like this: - // ... keep1 | dest=42 | keep2 | RF | magic | handler | magic | recursive args | - // Check various invariants. - verify_offsets(); - - Register rdi_temp = rdi; - Register rcx_temp = rcx; - { __ push(rdi_temp); __ push(rcx_temp); } -#define UNPUSH_TEMPS \ - { __ pop(rcx_temp); __ pop(rdi_temp); } - - Address magic_number_1_addr = RicochetFrame::frame_address(RicochetFrame::magic_number_1_offset_in_bytes()); - Address magic_number_2_addr = RicochetFrame::frame_address(RicochetFrame::magic_number_2_offset_in_bytes()); - Address continuation_addr = RicochetFrame::frame_address(RicochetFrame::continuation_offset_in_bytes()); - Address conversion_addr = RicochetFrame::frame_address(RicochetFrame::conversion_offset_in_bytes()); - Address saved_args_base_addr = RicochetFrame::frame_address(RicochetFrame::saved_args_base_offset_in_bytes()); - - Label L_bad, L_ok; - BLOCK_COMMENT("verify_clean {"); - // Magic numbers must check out: - __ cmpptr(magic_number_1_addr, (int32_t) MAGIC_NUMBER_1); - __ jcc(Assembler::notEqual, L_bad); - __ cmpptr(magic_number_2_addr, (int32_t) MAGIC_NUMBER_2); - __ jcc(Assembler::notEqual, L_bad); - - // Arguments pointer must look reasonable: - __ movptr(rcx_temp, saved_args_base_addr); - __ cmpptr(rcx_temp, rbp); - __ jcc(Assembler::below, L_bad); - __ subptr(rcx_temp, UNREASONABLE_STACK_MOVE * Interpreter::stackElementSize); - __ cmpptr(rcx_temp, rbp); - __ jcc(Assembler::above, L_bad); - - load_conversion_dest_type(_masm, rdi_temp, conversion_addr); - __ cmpl(rdi_temp, T_VOID); - __ jcc(Assembler::equal, L_ok); - __ movptr(rcx_temp, saved_args_base_addr); - load_conversion_vminfo(_masm, rdi_temp, conversion_addr); - __ cmpptr(Address(rcx_temp, rdi_temp, Interpreter::stackElementScale()), - (int32_t) RETURN_VALUE_PLACEHOLDER); - __ jcc(Assembler::equal, L_ok); - __ BIND(L_bad); - UNPUSH_TEMPS; - __ stop("damaged ricochet frame"); - __ BIND(L_ok); - UNPUSH_TEMPS; - BLOCK_COMMENT("} verify_clean"); - -#undef UNPUSH_TEMPS - -} -#endif //ASSERT - void MethodHandles::load_klass_from_Class(MacroAssembler* _masm, Register klass_reg) { if (VerifyMethodHandles) verify_klass(_masm, klass_reg, SystemDictionaryHandles::Class_klass(), - "AMH argument is a Class"); + "MH argument is a Class"); __ load_heap_oop(klass_reg, Address(klass_reg, java_lang_Class::klass_offset_in_bytes())); } -void MethodHandles::load_conversion_vminfo(MacroAssembler* _masm, Register reg, Address conversion_field_addr) { - int bits = BitsPerByte; - int offset = (CONV_VMINFO_SHIFT / bits); - int shift = (CONV_VMINFO_SHIFT % bits); - __ load_unsigned_byte(reg, conversion_field_addr.plus_disp(offset)); - assert(CONV_VMINFO_MASK == right_n_bits(bits - shift), "else change type of previous load"); - assert(shift == 0, "no shift needed"); -} - -void MethodHandles::load_conversion_dest_type(MacroAssembler* _masm, Register reg, Address conversion_field_addr) { - int bits = BitsPerByte; - int offset = (CONV_DEST_TYPE_SHIFT / bits); - int shift = (CONV_DEST_TYPE_SHIFT % bits); - __ load_unsigned_byte(reg, conversion_field_addr.plus_disp(offset)); - assert(CONV_TYPE_MASK == right_n_bits(bits - shift), "else change type of previous load"); - __ shrl(reg, shift); - DEBUG_ONLY(int conv_type_bits = (int) exact_log2(CONV_TYPE_MASK+1)); - assert((shift + conv_type_bits) == bits, "left justified in byte"); -} - -void MethodHandles::load_stack_move(MacroAssembler* _masm, - Register rdi_stack_move, - Register rcx_amh, - bool might_be_negative) { - BLOCK_COMMENT("load_stack_move {"); - Address rcx_amh_conversion(rcx_amh, java_lang_invoke_AdapterMethodHandle::conversion_offset_in_bytes()); - __ movl(rdi_stack_move, rcx_amh_conversion); - __ sarl(rdi_stack_move, CONV_STACK_MOVE_SHIFT); -#ifdef _LP64 - if (might_be_negative) { - // clean high bits of stack motion register (was loaded as an int) - __ movslq(rdi_stack_move, rdi_stack_move); - } -#endif //_LP64 -#ifdef ASSERT - if (VerifyMethodHandles) { - Label L_ok, L_bad; - int32_t stack_move_limit = 0x4000; // extra-large - __ cmpptr(rdi_stack_move, stack_move_limit); - __ jcc(Assembler::greaterEqual, L_bad); - __ cmpptr(rdi_stack_move, -stack_move_limit); - __ jcc(Assembler::greater, L_ok); - __ bind(L_bad); - __ stop("load_stack_move of garbage value"); - __ BIND(L_ok); - } -#endif - BLOCK_COMMENT("} load_stack_move"); -} - #ifdef ASSERT -void MethodHandles::RicochetFrame::verify_offsets() { - // Check compatibility of this struct with the more generally used offsets of class frame: - int ebp_off = sender_link_offset_in_bytes(); // offset from struct base to local rbp value - assert(ebp_off + wordSize*frame::interpreter_frame_method_offset == saved_args_base_offset_in_bytes(), ""); - assert(ebp_off + wordSize*frame::interpreter_frame_last_sp_offset == conversion_offset_in_bytes(), ""); - assert(ebp_off + wordSize*frame::interpreter_frame_sender_sp_offset == exact_sender_sp_offset_in_bytes(), ""); - // These last two have to be exact: - assert(ebp_off + wordSize*frame::link_offset == sender_link_offset_in_bytes(), ""); - assert(ebp_off + wordSize*frame::return_addr_offset == sender_pc_offset_in_bytes(), ""); +static int check_nonzero(const char* xname, int x) { + assert(x != 0, err_msg("%s should be nonzero", xname)); + return x; } - -void MethodHandles::RicochetFrame::verify() const { - verify_offsets(); - assert(magic_number_1() == MAGIC_NUMBER_1, err_msg(PTR_FORMAT " == " PTR_FORMAT, magic_number_1(), MAGIC_NUMBER_1)); - assert(magic_number_2() == MAGIC_NUMBER_2, err_msg(PTR_FORMAT " == " PTR_FORMAT, magic_number_2(), MAGIC_NUMBER_2)); - if (!Universe::heap()->is_gc_active()) { - if (saved_args_layout() != NULL) { - assert(saved_args_layout()->is_method(), "must be valid oop"); - } - if (saved_target() != NULL) { - assert(java_lang_invoke_MethodHandle::is_instance(saved_target()), "checking frame value"); - } - } - int conv_op = adapter_conversion_op(conversion()); - assert(conv_op == java_lang_invoke_AdapterMethodHandle::OP_COLLECT_ARGS || - conv_op == java_lang_invoke_AdapterMethodHandle::OP_FOLD_ARGS || - conv_op == java_lang_invoke_AdapterMethodHandle::OP_PRIM_TO_REF, - "must be a sane conversion"); - if (has_return_value_slot()) { - assert(*return_value_slot_addr() == RETURN_VALUE_PLACEHOLDER, ""); - } -} -#endif //PRODUCT +#define NONZERO(x) check_nonzero(#x, x) +#else //ASSERT +#define NONZERO(x) (x) +#endif //ASSERT #ifdef ASSERT -void MethodHandles::verify_argslot(MacroAssembler* _masm, - Register argslot_reg, - const char* error_message) { - // Verify that argslot lies within (rsp, rbp]. - Label L_ok, L_bad; - BLOCK_COMMENT("verify_argslot {"); - __ cmpptr(argslot_reg, rbp); - __ jccb(Assembler::above, L_bad); - __ cmpptr(rsp, argslot_reg); - __ jccb(Assembler::below, L_ok); - __ bind(L_bad); - __ stop(error_message); - __ BIND(L_ok); - BLOCK_COMMENT("} verify_argslot"); -} - -void MethodHandles::verify_argslots(MacroAssembler* _masm, - RegisterOrConstant arg_slots, - Register arg_slot_base_reg, - bool negate_argslots, - const char* error_message) { - // Verify that [argslot..argslot+size) lies within (rsp, rbp). - Label L_ok, L_bad; - Register rdi_temp = rdi; - BLOCK_COMMENT("verify_argslots {"); - __ push(rdi_temp); - if (negate_argslots) { - if (arg_slots.is_constant()) { - arg_slots = -1 * arg_slots.as_constant(); - } else { - __ movptr(rdi_temp, arg_slots); - __ negptr(rdi_temp); - arg_slots = rdi_temp; - } - } - __ lea(rdi_temp, Address(arg_slot_base_reg, arg_slots, Interpreter::stackElementScale())); - __ cmpptr(rdi_temp, rbp); - __ pop(rdi_temp); - __ jcc(Assembler::above, L_bad); - __ cmpptr(rsp, arg_slot_base_reg); - __ jcc(Assembler::below, L_ok); - __ bind(L_bad); - __ stop(error_message); - __ BIND(L_ok); - BLOCK_COMMENT("} verify_argslots"); -} - -// Make sure that arg_slots has the same sign as the given direction. -// If (and only if) arg_slots is a assembly-time constant, also allow it to be zero. -void MethodHandles::verify_stack_move(MacroAssembler* _masm, - RegisterOrConstant arg_slots, int direction) { - bool allow_zero = arg_slots.is_constant(); - if (direction == 0) { direction = +1; allow_zero = true; } - assert(stack_move_unit() == -1, "else add extra checks here"); - if (arg_slots.is_register()) { - Label L_ok, L_bad; - BLOCK_COMMENT("verify_stack_move {"); - // testl(arg_slots.as_register(), -stack_move_unit() - 1); // no need - // jcc(Assembler::notZero, L_bad); - __ cmpptr(arg_slots.as_register(), (int32_t) NULL_WORD); - if (direction > 0) { - __ jcc(allow_zero ? Assembler::less : Assembler::lessEqual, L_bad); - __ cmpptr(arg_slots.as_register(), (int32_t) UNREASONABLE_STACK_MOVE); - __ jcc(Assembler::less, L_ok); - } else { - __ jcc(allow_zero ? Assembler::greater : Assembler::greaterEqual, L_bad); - __ cmpptr(arg_slots.as_register(), (int32_t) -UNREASONABLE_STACK_MOVE); - __ jcc(Assembler::greater, L_ok); - } - __ bind(L_bad); - if (direction > 0) - __ stop("assert arg_slots > 0"); - else - __ stop("assert arg_slots < 0"); - __ BIND(L_ok); - BLOCK_COMMENT("} verify_stack_move"); - } else { - intptr_t size = arg_slots.as_constant(); - if (direction < 0) size = -size; - assert(size >= 0, "correct direction of constant move"); - assert(size < UNREASONABLE_STACK_MOVE, "reasonable size of constant move"); - } -} - void MethodHandles::verify_klass(MacroAssembler* _masm, Register obj, KlassHandle klass, const char* error_message) { @@ -528,12 +71,15 @@ klass_addr <= SystemDictionaryHandles::Long_klass().raw_value(), "must be one of the SystemDictionaryHandles"); Register temp = rdi; + Register temp2 = noreg; + LP64_ONLY(temp2 = rscratch1); // used by MacroAssembler::cmpptr Label L_ok, L_bad; BLOCK_COMMENT("verify_klass {"); __ verify_oop(obj); __ testptr(obj, obj); __ jcc(Assembler::zero, L_bad); - __ push(temp); + __ push(temp); if (temp2 != noreg) __ push(temp2); +#define UNPUSH { if (temp2 != noreg) __ pop(temp2); __ pop(temp); } __ load_klass(temp, obj); __ cmpptr(temp, ExternalAddress((address) klass_addr)); __ jcc(Assembler::equal, L_ok); @@ -541,17 +87,42 @@ __ movptr(temp, Address(temp, super_check_offset)); __ cmpptr(temp, ExternalAddress((address) klass_addr)); __ jcc(Assembler::equal, L_ok); - __ pop(temp); + UNPUSH; __ bind(L_bad); - __ stop(error_message); + __ STOP(error_message); __ BIND(L_ok); - __ pop(temp); + UNPUSH; BLOCK_COMMENT("} verify_klass"); } + +void MethodHandles::verify_ref_kind(MacroAssembler* _masm, int ref_kind, Register member_reg, Register temp) { + Label L; + BLOCK_COMMENT("verify_ref_kind {"); + __ movl(temp, Address(member_reg, NONZERO(java_lang_invoke_MemberName::flags_offset_in_bytes()))); + __ shrl(temp, java_lang_invoke_MemberName::MN_REFERENCE_KIND_SHIFT); + __ andl(temp, java_lang_invoke_MemberName::MN_REFERENCE_KIND_MASK); + __ cmpl(temp, ref_kind); + __ jcc(Assembler::equal, L); + { char* buf = NEW_C_HEAP_ARRAY(char, 100, mtInternal); + jio_snprintf(buf, 100, "verify_ref_kind expected %x", ref_kind); + if (ref_kind == JVM_REF_invokeVirtual || + ref_kind == JVM_REF_invokeSpecial) + // could do this for all ref_kinds, but would explode assembly code size + trace_method_handle(_masm, buf); + __ STOP(buf); + } + BLOCK_COMMENT("} verify_ref_kind"); + __ bind(L); +} + #endif //ASSERT -void MethodHandles::jump_from_method_handle(MacroAssembler* _masm, Register method, Register temp) { - if (JvmtiExport::can_post_interpreter_events()) { +void MethodHandles::jump_from_method_handle(MacroAssembler* _masm, Register method, Register temp, + bool for_compiler_entry) { + assert(method == rbx, "interpreter calling convention"); + __ verify_oop(method); + + if (!for_compiler_entry && JvmtiExport::can_post_interpreter_events()) { Label run_compiled_code; // JVMTI events, such as single-stepping, are implemented partly by avoiding running // compiled code in threads for which the event is enabled. Check here for @@ -567,462 +138,380 @@ __ cmpb(Address(rthread, JavaThread::interp_only_mode_offset()), 0); __ jccb(Assembler::zero, run_compiled_code); __ jmp(Address(method, methodOopDesc::interpreter_entry_offset())); - __ bind(run_compiled_code); + __ BIND(run_compiled_code); } - __ jmp(Address(method, methodOopDesc::from_interpreted_offset())); + + const ByteSize entry_offset = for_compiler_entry ? methodOopDesc::from_compiled_offset() : + methodOopDesc::from_interpreted_offset(); + __ jmp(Address(method, entry_offset)); } +void MethodHandles::jump_to_lambda_form(MacroAssembler* _masm, + Register recv, Register method_temp, + Register temp2, + bool for_compiler_entry) { + BLOCK_COMMENT("jump_to_lambda_form {"); + // This is the initial entry point of a lazy method handle. + // After type checking, it picks up the invoker from the LambdaForm. + assert_different_registers(recv, method_temp, temp2); + assert(recv != noreg, "required register"); + assert(method_temp == rbx, "required register for loading method"); + + //NOT_PRODUCT({ FlagSetting fs(TraceMethodHandles, true); trace_method_handle(_masm, "LZMH"); }); + + // Load the invoker, as MH -> MH.form -> LF.vmentry + __ verify_oop(recv); + __ load_heap_oop(method_temp, Address(recv, NONZERO(java_lang_invoke_MethodHandle::form_offset_in_bytes()))); + __ verify_oop(method_temp); + __ load_heap_oop(method_temp, Address(method_temp, NONZERO(java_lang_invoke_LambdaForm::vmentry_offset_in_bytes()))); + __ verify_oop(method_temp); + // the following assumes that a methodOop is normally compressed in the vmtarget field: + __ load_heap_oop(method_temp, Address(method_temp, NONZERO(java_lang_invoke_MemberName::vmtarget_offset_in_bytes()))); + __ verify_oop(method_temp); + + if (VerifyMethodHandles && !for_compiler_entry) { + // make sure recv is already on stack + __ load_sized_value(temp2, + Address(method_temp, methodOopDesc::size_of_parameters_offset()), + sizeof(u2), /*is_signed*/ false); + // assert(sizeof(u2) == sizeof(methodOopDesc::_size_of_parameters), ""); + Label L; + __ cmpptr(recv, __ argument_address(temp2, -1)); + __ jcc(Assembler::equal, L); + __ movptr(rax, __ argument_address(temp2, -1)); + __ STOP("receiver not on stack"); + __ BIND(L); + } + + jump_from_method_handle(_masm, method_temp, temp2, for_compiler_entry); + BLOCK_COMMENT("} jump_to_lambda_form"); +} + + // Code generation -address MethodHandles::generate_method_handle_interpreter_entry(MacroAssembler* _masm) { - // rbx: methodOop - // rcx: receiver method handle (must load from sp[MethodTypeForm.vmslots]) - // rsi/r13: sender SP (must preserve; see prepare_to_jump_from_interpreted) - // rdx, rdi: garbage temp, blown away +address MethodHandles::generate_method_handle_interpreter_entry(MacroAssembler* _masm, + vmIntrinsics::ID iid) { + const bool not_for_compiler_entry = false; // this is the interpreter entry + assert(is_signature_polymorphic(iid), "expected invoke iid"); + if (iid == vmIntrinsics::_invokeGeneric || + iid == vmIntrinsics::_compiledLambdaForm) { + // Perhaps surprisingly, the symbolic references visible to Java are not directly used. + // They are linked to Java-generated adapters via MethodHandleNatives.linkMethod. + // They all allow an appendix argument. + __ hlt(); // empty stubs make SG sick + return NULL; + } - Register rbx_method = rbx; - Register rcx_recv = rcx; - Register rax_mtype = rax; - Register rdx_temp = rdx; - Register rdi_temp = rdi; + // rsi/r13: sender SP (must preserve; see prepare_to_jump_from_interpreted) + // rbx: methodOop + // rdx: argument locator (parameter slot count, added to rsp) + // rcx: used as temp to hold mh or receiver + // rax, rdi: garbage temps, blown away + Register rdx_argp = rdx; // argument list ptr, live on error paths + Register rax_temp = rax; + Register rcx_mh = rcx; // MH receiver; dies quickly and is recycled + Register rbx_method = rbx; // eventual target of this invocation - // emit WrongMethodType path first, to enable jccb back-branch from main path - Label wrong_method_type; - __ bind(wrong_method_type); - Label invoke_generic_slow_path, invoke_exact_error_path; - assert(methodOopDesc::intrinsic_id_size_in_bytes() == sizeof(u1), "");; - __ cmpb(Address(rbx_method, methodOopDesc::intrinsic_id_offset_in_bytes()), (int) vmIntrinsics::_invokeExact); - __ jcc(Assembler::notEqual, invoke_generic_slow_path); - __ jmp(invoke_exact_error_path); + address code_start = __ pc(); // here's where control starts out: __ align(CodeEntryAlignment); address entry_point = __ pc(); - // fetch the MethodType from the method handle into rax (the 'check' register) - // FIXME: Interpreter should transmit pre-popped stack pointer, to locate base of arg list. - // This would simplify several touchy bits of code. - // See 6984712: JSR 292 method handle calls need a clean argument base pointer - { - Register tem = rbx_method; - for (jint* pchase = methodOopDesc::method_type_offsets_chain(); (*pchase) != -1; pchase++) { - __ movptr(rax_mtype, Address(tem, *pchase)); - tem = rax_mtype; // in case there is another indirection + if (VerifyMethodHandles) { + Label L; + BLOCK_COMMENT("verify_intrinsic_id {"); + __ cmpb(Address(rbx_method, methodOopDesc::intrinsic_id_offset_in_bytes()), (int) iid); + __ jcc(Assembler::equal, L); + if (iid == vmIntrinsics::_linkToVirtual || + iid == vmIntrinsics::_linkToSpecial) { + // could do this for all kinds, but would explode assembly code size + trace_method_handle(_masm, "bad methodOop::intrinsic_id"); } + __ STOP("bad methodOop::intrinsic_id"); + __ bind(L); + BLOCK_COMMENT("} verify_intrinsic_id"); + } + + // First task: Find out how big the argument list is. + Address rdx_first_arg_addr; + int ref_kind = signature_polymorphic_intrinsic_ref_kind(iid); + assert(ref_kind != 0 || iid == vmIntrinsics::_invokeBasic, "must be _invokeBasic or a linkTo intrinsic"); + if (ref_kind == 0 || MethodHandles::ref_kind_has_receiver(ref_kind)) { + __ load_sized_value(rdx_argp, + Address(rbx_method, methodOopDesc::size_of_parameters_offset()), + sizeof(u2), /*is_signed*/ false); + // assert(sizeof(u2) == sizeof(methodOopDesc::_size_of_parameters), ""); + rdx_first_arg_addr = __ argument_address(rdx_argp, -1); + } else { + DEBUG_ONLY(rdx_argp = noreg); + } + + if (!is_signature_polymorphic_static(iid)) { + __ movptr(rcx_mh, rdx_first_arg_addr); + DEBUG_ONLY(rdx_argp = noreg); } - // given the MethodType, find out where the MH argument is buried - __ load_heap_oop(rdx_temp, Address(rax_mtype, __ delayed_value(java_lang_invoke_MethodType::form_offset_in_bytes, rdi_temp))); - Register rdx_vmslots = rdx_temp; - __ movl(rdx_vmslots, Address(rdx_temp, __ delayed_value(java_lang_invoke_MethodTypeForm::vmslots_offset_in_bytes, rdi_temp))); - Address mh_receiver_slot_addr = __ argument_address(rdx_vmslots); - __ movptr(rcx_recv, mh_receiver_slot_addr); - - trace_method_handle(_masm, "invokeExact"); - - __ check_method_handle_type(rax_mtype, rcx_recv, rdi_temp, wrong_method_type); + // rdx_first_arg_addr is live! - // Nobody uses the MH receiver slot after this. Make sure. - DEBUG_ONLY(__ movptr(mh_receiver_slot_addr, (int32_t)0x999999)); - - __ jump_to_method_handle_entry(rcx_recv, rdi_temp); + if (TraceMethodHandles) { + const char* name = vmIntrinsics::name_at(iid); + if (*name == '_') name += 1; + const size_t len = strlen(name) + 50; + char* qname = NEW_C_HEAP_ARRAY(char, len, mtInternal); + const char* suffix = ""; + if (vmIntrinsics::method_for(iid) == NULL || + !vmIntrinsics::method_for(iid)->access_flags().is_public()) { + if (is_signature_polymorphic_static(iid)) + suffix = "/static"; + else + suffix = "/private"; + } + jio_snprintf(qname, len, "MethodHandle::interpreter_entry::%s%s", name, suffix); + // note: stub look for mh in rcx + trace_method_handle(_masm, qname); + } - // error path for invokeExact (only) - __ bind(invoke_exact_error_path); - // ensure that the top of stack is properly aligned. - __ mov(rdi, rsp); - __ andptr(rsp, -StackAlignmentInBytes); // Align the stack for the ABI - __ pushptr(Address(rdi, 0)); // Pick up the return address - - // Stub wants expected type in rax and the actual type in rcx - __ jump(ExternalAddress(StubRoutines::throw_WrongMethodTypeException_entry())); + if (iid == vmIntrinsics::_invokeBasic) { + generate_method_handle_dispatch(_masm, iid, rcx_mh, noreg, not_for_compiler_entry); - // for invokeGeneric (only), apply argument and result conversions on the fly - __ bind(invoke_generic_slow_path); -#ifdef ASSERT - if (VerifyMethodHandles) { - Label L; - __ cmpb(Address(rbx_method, methodOopDesc::intrinsic_id_offset_in_bytes()), (int) vmIntrinsics::_invokeGeneric); - __ jcc(Assembler::equal, L); - __ stop("bad methodOop::intrinsic_id"); - __ bind(L); + } else { + // Adjust argument list by popping the trailing MemberName argument. + Register rcx_recv = noreg; + if (MethodHandles::ref_kind_has_receiver(ref_kind)) { + // Load the receiver (not the MH; the actual MemberName's receiver) up from the interpreter stack. + __ movptr(rcx_recv = rcx, rdx_first_arg_addr); + } + DEBUG_ONLY(rdx_argp = noreg); + Register rbx_member = rbx_method; // MemberName ptr; incoming method ptr is dead now + __ pop(rax_temp); // return address + __ pop(rbx_member); // extract last argument + __ push(rax_temp); // re-push return address + generate_method_handle_dispatch(_masm, iid, rcx_recv, rbx_member, not_for_compiler_entry); } -#endif //ASSERT - Register rbx_temp = rbx_method; // don't need it now - // make room on the stack for another pointer: - Register rcx_argslot = rcx_recv; - __ lea(rcx_argslot, __ argument_address(rdx_vmslots, 1)); - insert_arg_slots(_masm, 2 * stack_move_unit(), - rcx_argslot, rbx_temp, rdx_temp); - - // load up an adapter from the calling type (Java weaves this) - Register rdx_adapter = rdx_temp; - __ load_heap_oop(rdx_temp, Address(rax_mtype, __ delayed_value(java_lang_invoke_MethodType::form_offset_in_bytes, rdi_temp))); - __ load_heap_oop(rdx_adapter, Address(rdx_temp, __ delayed_value(java_lang_invoke_MethodTypeForm::genericInvoker_offset_in_bytes, rdi_temp))); - __ verify_oop(rdx_adapter); - __ movptr(Address(rcx_argslot, 1 * Interpreter::stackElementSize), rdx_adapter); - // As a trusted first argument, pass the type being called, so the adapter knows - // the actual types of the arguments and return values. - // (Generic invokers are shared among form-families of method-type.) - __ movptr(Address(rcx_argslot, 0 * Interpreter::stackElementSize), rax_mtype); - // FIXME: assert that rdx_adapter is of the right method-type. - __ mov(rcx, rdx_adapter); - trace_method_handle(_masm, "invokeGeneric"); - __ jump_to_method_handle_entry(rcx, rdi_temp); + if (PrintMethodHandleStubs) { + address code_end = __ pc(); + tty->print_cr("--------"); + tty->print_cr("method handle interpreter entry for %s", vmIntrinsics::name_at(iid)); + Disassembler::decode(code_start, code_end); + tty->cr(); + } return entry_point; } -// Helper to insert argument slots into the stack. -// arg_slots must be a multiple of stack_move_unit() and < 0 -// rax_argslot is decremented to point to the new (shifted) location of the argslot -// But, rdx_temp ends up holding the original value of rax_argslot. -void MethodHandles::insert_arg_slots(MacroAssembler* _masm, - RegisterOrConstant arg_slots, - Register rax_argslot, - Register rbx_temp, Register rdx_temp) { - // allow constant zero - if (arg_slots.is_constant() && arg_slots.as_constant() == 0) - return; - assert_different_registers(rax_argslot, rbx_temp, rdx_temp, - (!arg_slots.is_register() ? rsp : arg_slots.as_register())); - if (VerifyMethodHandles) - verify_argslot(_masm, rax_argslot, "insertion point must fall within current frame"); - if (VerifyMethodHandles) - verify_stack_move(_masm, arg_slots, -1); - - // Make space on the stack for the inserted argument(s). - // Then pull down everything shallower than rax_argslot. - // The stacked return address gets pulled down with everything else. - // That is, copy [rsp, argslot) downward by -size words. In pseudo-code: - // rsp -= size; - // for (rdx = rsp + size; rdx < argslot; rdx++) - // rdx[-size] = rdx[0] - // argslot -= size; - BLOCK_COMMENT("insert_arg_slots {"); - __ mov(rdx_temp, rsp); // source pointer for copy - __ lea(rsp, Address(rsp, arg_slots, Interpreter::stackElementScale())); - { - Label loop; - __ BIND(loop); - // pull one word down each time through the loop - __ movptr(rbx_temp, Address(rdx_temp, 0)); - __ movptr(Address(rdx_temp, arg_slots, Interpreter::stackElementScale()), rbx_temp); - __ addptr(rdx_temp, wordSize); - __ cmpptr(rdx_temp, rax_argslot); - __ jcc(Assembler::below, loop); - } - - // Now move the argslot down, to point to the opened-up space. - __ lea(rax_argslot, Address(rax_argslot, arg_slots, Interpreter::stackElementScale())); - BLOCK_COMMENT("} insert_arg_slots"); -} - -// Helper to remove argument slots from the stack. -// arg_slots must be a multiple of stack_move_unit() and > 0 -void MethodHandles::remove_arg_slots(MacroAssembler* _masm, - RegisterOrConstant arg_slots, - Register rax_argslot, - Register rbx_temp, Register rdx_temp) { - // allow constant zero - if (arg_slots.is_constant() && arg_slots.as_constant() == 0) - return; - assert_different_registers(rax_argslot, rbx_temp, rdx_temp, - (!arg_slots.is_register() ? rsp : arg_slots.as_register())); - if (VerifyMethodHandles) - verify_argslots(_masm, arg_slots, rax_argslot, false, - "deleted argument(s) must fall within current frame"); - if (VerifyMethodHandles) - verify_stack_move(_masm, arg_slots, +1); - - BLOCK_COMMENT("remove_arg_slots {"); - // Pull up everything shallower than rax_argslot. - // Then remove the excess space on the stack. - // The stacked return address gets pulled up with everything else. - // That is, copy [rsp, argslot) upward by size words. In pseudo-code: - // for (rdx = argslot-1; rdx >= rsp; --rdx) - // rdx[size] = rdx[0] - // argslot += size; - // rsp += size; - __ lea(rdx_temp, Address(rax_argslot, -wordSize)); // source pointer for copy - { - Label loop; - __ BIND(loop); - // pull one word up each time through the loop - __ movptr(rbx_temp, Address(rdx_temp, 0)); - __ movptr(Address(rdx_temp, arg_slots, Interpreter::stackElementScale()), rbx_temp); - __ addptr(rdx_temp, -wordSize); - __ cmpptr(rdx_temp, rsp); - __ jcc(Assembler::aboveEqual, loop); +void MethodHandles::generate_method_handle_dispatch(MacroAssembler* _masm, + vmIntrinsics::ID iid, + Register receiver_reg, + Register member_reg, + bool for_compiler_entry) { + assert(is_signature_polymorphic(iid), "expected invoke iid"); + Register rbx_method = rbx; // eventual target of this invocation + // temps used in this code are not used in *either* compiled or interpreted calling sequences +#ifdef _LP64 + Register temp1 = rscratch1; + Register temp2 = rscratch2; + Register temp3 = rax; + if (for_compiler_entry) { + assert(receiver_reg == (iid == vmIntrinsics::_linkToStatic ? noreg : j_rarg0), "only valid assignment"); + assert_different_registers(temp1, j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5); + assert_different_registers(temp2, j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5); + assert_different_registers(temp3, j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5); } - - // Now move the argslot up, to point to the just-copied block. - __ lea(rsp, Address(rsp, arg_slots, Interpreter::stackElementScale())); - // And adjust the argslot address to point at the deletion point. - __ lea(rax_argslot, Address(rax_argslot, arg_slots, Interpreter::stackElementScale())); - BLOCK_COMMENT("} remove_arg_slots"); -} - -// Helper to copy argument slots to the top of the stack. -// The sequence starts with rax_argslot and is counted by slot_count -// slot_count must be a multiple of stack_move_unit() and >= 0 -// This function blows the temps but does not change rax_argslot. -void MethodHandles::push_arg_slots(MacroAssembler* _masm, - Register rax_argslot, - RegisterOrConstant slot_count, - int skip_words_count, - Register rbx_temp, Register rdx_temp) { - assert_different_registers(rax_argslot, rbx_temp, rdx_temp, - (!slot_count.is_register() ? rbp : slot_count.as_register()), - rsp); - assert(Interpreter::stackElementSize == wordSize, "else change this code"); - - if (VerifyMethodHandles) - verify_stack_move(_masm, slot_count, 0); - - // allow constant zero - if (slot_count.is_constant() && slot_count.as_constant() == 0) - return; - - BLOCK_COMMENT("push_arg_slots {"); - - Register rbx_top = rbx_temp; - - // There is at most 1 word to carry down with the TOS. - switch (skip_words_count) { - case 1: __ pop(rdx_temp); break; - case 0: break; - default: ShouldNotReachHere(); - } - - if (slot_count.is_constant()) { - for (int i = slot_count.as_constant() - 1; i >= 0; i--) { - __ pushptr(Address(rax_argslot, i * wordSize)); - } - } else { - Label L_plural, L_loop, L_break; - // Emit code to dynamically check for the common cases, zero and one slot. - __ cmpl(slot_count.as_register(), (int32_t) 1); - __ jccb(Assembler::greater, L_plural); - __ jccb(Assembler::less, L_break); - __ pushptr(Address(rax_argslot, 0)); - __ jmpb(L_break); - __ BIND(L_plural); - - // Loop for 2 or more: - // rbx = &rax[slot_count] - // while (rbx > rax) *(--rsp) = *(--rbx) - __ lea(rbx_top, Address(rax_argslot, slot_count, Address::times_ptr)); - __ BIND(L_loop); - __ subptr(rbx_top, wordSize); - __ pushptr(Address(rbx_top, 0)); - __ cmpptr(rbx_top, rax_argslot); - __ jcc(Assembler::above, L_loop); - __ bind(L_break); - } - switch (skip_words_count) { - case 1: __ push(rdx_temp); break; - case 0: break; - default: ShouldNotReachHere(); - } - BLOCK_COMMENT("} push_arg_slots"); -} - -// in-place movement; no change to rsp -// blows rax_temp, rdx_temp -void MethodHandles::move_arg_slots_up(MacroAssembler* _masm, - Register rbx_bottom, // invariant - Address top_addr, // can use rax_temp - RegisterOrConstant positive_distance_in_slots, - Register rax_temp, Register rdx_temp) { - BLOCK_COMMENT("move_arg_slots_up {"); - assert_different_registers(rbx_bottom, - rax_temp, rdx_temp, - positive_distance_in_slots.register_or_noreg()); - Label L_loop, L_break; - Register rax_top = rax_temp; - if (!top_addr.is_same_address(Address(rax_top, 0))) - __ lea(rax_top, top_addr); - // Detect empty (or broken) loop: -#ifdef ASSERT - if (VerifyMethodHandles) { - // Verify that &bottom < &top (non-empty interval) - Label L_ok, L_bad; - if (positive_distance_in_slots.is_register()) { - __ cmpptr(positive_distance_in_slots.as_register(), (int32_t) 0); - __ jcc(Assembler::lessEqual, L_bad); - } - __ cmpptr(rbx_bottom, rax_top); - __ jcc(Assembler::below, L_ok); - __ bind(L_bad); - __ stop("valid bounds (copy up)"); - __ BIND(L_ok); +#else + Register temp1 = (for_compiler_entry ? rsi : rdx); + Register temp2 = rdi; + Register temp3 = rax; + if (for_compiler_entry) { + assert(receiver_reg == (iid == vmIntrinsics::_linkToStatic ? noreg : rcx), "only valid assignment"); + assert_different_registers(temp1, rcx, rdx); + assert_different_registers(temp2, rcx, rdx); + assert_different_registers(temp3, rcx, rdx); } #endif - __ cmpptr(rbx_bottom, rax_top); - __ jccb(Assembler::aboveEqual, L_break); - // work rax down to rbx, copying contiguous data upwards - // In pseudo-code: - // [rbx, rax) = &[bottom, top) - // while (--rax >= rbx) *(rax + distance) = *(rax + 0), rax--; - __ BIND(L_loop); - __ subptr(rax_top, wordSize); - __ movptr(rdx_temp, Address(rax_top, 0)); - __ movptr( Address(rax_top, positive_distance_in_slots, Address::times_ptr), rdx_temp); - __ cmpptr(rax_top, rbx_bottom); - __ jcc(Assembler::above, L_loop); - assert(Interpreter::stackElementSize == wordSize, "else change loop"); - __ bind(L_break); - BLOCK_COMMENT("} move_arg_slots_up"); -} + assert_different_registers(temp1, temp2, temp3, receiver_reg); + assert_different_registers(temp1, temp2, temp3, member_reg); + if (!for_compiler_entry) + assert_different_registers(temp1, temp2, temp3, saved_last_sp_register()); // don't trash lastSP + + if (iid == vmIntrinsics::_invokeBasic) { + // indirect through MH.form.vmentry.vmtarget + jump_to_lambda_form(_masm, receiver_reg, rbx_method, temp1, for_compiler_entry); + + } else { + // The method is a member invoker used by direct method handles. + if (VerifyMethodHandles) { + // make sure the trailing argument really is a MemberName (caller responsibility) + verify_klass(_masm, member_reg, SystemDictionaryHandles::MemberName_klass(), + "MemberName required for invokeVirtual etc."); + } + + Address member_clazz( member_reg, NONZERO(java_lang_invoke_MemberName::clazz_offset_in_bytes())); + Address member_vmindex( member_reg, NONZERO(java_lang_invoke_MemberName::vmindex_offset_in_bytes())); + Address member_vmtarget( member_reg, NONZERO(java_lang_invoke_MemberName::vmtarget_offset_in_bytes())); -// in-place movement; no change to rsp -// blows rax_temp, rdx_temp -void MethodHandles::move_arg_slots_down(MacroAssembler* _masm, - Address bottom_addr, // can use rax_temp - Register rbx_top, // invariant - RegisterOrConstant negative_distance_in_slots, - Register rax_temp, Register rdx_temp) { - BLOCK_COMMENT("move_arg_slots_down {"); - assert_different_registers(rbx_top, - negative_distance_in_slots.register_or_noreg(), - rax_temp, rdx_temp); - Label L_loop, L_break; - Register rax_bottom = rax_temp; - if (!bottom_addr.is_same_address(Address(rax_bottom, 0))) - __ lea(rax_bottom, bottom_addr); - // Detect empty (or broken) loop: -#ifdef ASSERT - assert(!negative_distance_in_slots.is_constant() || negative_distance_in_slots.as_constant() < 0, ""); - if (VerifyMethodHandles) { - // Verify that &bottom < &top (non-empty interval) - Label L_ok, L_bad; - if (negative_distance_in_slots.is_register()) { - __ cmpptr(negative_distance_in_slots.as_register(), (int32_t) 0); - __ jcc(Assembler::greaterEqual, L_bad); + Register temp1_recv_klass = temp1; + if (iid != vmIntrinsics::_linkToStatic) { + __ verify_oop(receiver_reg); + if (iid == vmIntrinsics::_linkToSpecial) { + // Don't actually load the klass; just null-check the receiver. + __ null_check(receiver_reg); + } else { + // load receiver klass itself + __ null_check(receiver_reg, oopDesc::klass_offset_in_bytes()); + __ load_klass(temp1_recv_klass, receiver_reg); + __ verify_oop(temp1_recv_klass); + } + BLOCK_COMMENT("check_receiver {"); + // The receiver for the MemberName must be in receiver_reg. + // Check the receiver against the MemberName.clazz + if (VerifyMethodHandles && iid == vmIntrinsics::_linkToSpecial) { + // Did not load it above... + __ load_klass(temp1_recv_klass, receiver_reg); + __ verify_oop(temp1_recv_klass); + } + if (VerifyMethodHandles && iid != vmIntrinsics::_linkToInterface) { + Label L_ok; + Register temp2_defc = temp2; + __ load_heap_oop(temp2_defc, member_clazz); + load_klass_from_Class(_masm, temp2_defc); + __ verify_oop(temp2_defc); + __ check_klass_subtype(temp1_recv_klass, temp2_defc, temp3, L_ok); + // If we get here, the type check failed! + __ STOP("receiver class disagrees with MemberName.clazz"); + __ bind(L_ok); + } + BLOCK_COMMENT("} check_receiver"); } - __ cmpptr(rax_bottom, rbx_top); - __ jcc(Assembler::below, L_ok); - __ bind(L_bad); - __ stop("valid bounds (copy down)"); - __ BIND(L_ok); - } -#endif - __ cmpptr(rax_bottom, rbx_top); - __ jccb(Assembler::aboveEqual, L_break); - // work rax up to rbx, copying contiguous data downwards - // In pseudo-code: - // [rax, rbx) = &[bottom, top) - // while (rax < rbx) *(rax - distance) = *(rax + 0), rax++; - __ BIND(L_loop); - __ movptr(rdx_temp, Address(rax_bottom, 0)); - __ movptr( Address(rax_bottom, negative_distance_in_slots, Address::times_ptr), rdx_temp); - __ addptr(rax_bottom, wordSize); - __ cmpptr(rax_bottom, rbx_top); - __ jcc(Assembler::below, L_loop); - assert(Interpreter::stackElementSize == wordSize, "else change loop"); - __ bind(L_break); - BLOCK_COMMENT("} move_arg_slots_down"); -} + if (iid == vmIntrinsics::_linkToSpecial || + iid == vmIntrinsics::_linkToStatic) { + DEBUG_ONLY(temp1_recv_klass = noreg); // these guys didn't load the recv_klass + } + + // Live registers at this point: + // member_reg - MemberName that was the trailing argument + // temp1_recv_klass - klass of stacked receiver, if needed + // rsi/r13 - interpreter linkage (if interpreted) + // rcx, rdx, rsi, rdi, r8, r8 - compiler arguments (if compiled) + + bool method_is_live = false; + switch (iid) { + case vmIntrinsics::_linkToSpecial: + if (VerifyMethodHandles) { + verify_ref_kind(_masm, JVM_REF_invokeSpecial, member_reg, temp3); + } + __ load_heap_oop(rbx_method, member_vmtarget); + method_is_live = true; + break; + + case vmIntrinsics::_linkToStatic: + if (VerifyMethodHandles) { + verify_ref_kind(_masm, JVM_REF_invokeStatic, member_reg, temp3); + } + __ load_heap_oop(rbx_method, member_vmtarget); + method_is_live = true; + break; -// Copy from a field or array element to a stacked argument slot. -// is_element (ignored) says whether caller is loading an array element instead of an instance field. -void MethodHandles::move_typed_arg(MacroAssembler* _masm, - BasicType type, bool is_element, - Address slot_dest, Address value_src, - Register rbx_temp, Register rdx_temp) { - BLOCK_COMMENT(!is_element ? "move_typed_arg {" : "move_typed_arg { (array element)"); - if (type == T_OBJECT || type == T_ARRAY) { - __ load_heap_oop(rbx_temp, value_src); - __ movptr(slot_dest, rbx_temp); - } else if (type != T_VOID) { - int arg_size = type2aelembytes(type); - bool arg_is_signed = is_signed_subword_type(type); - int slot_size = (arg_size > wordSize) ? arg_size : wordSize; - __ load_sized_value( rdx_temp, value_src, arg_size, arg_is_signed, rbx_temp); - __ store_sized_value( slot_dest, rdx_temp, slot_size, rbx_temp); - } - BLOCK_COMMENT("} move_typed_arg"); -} + case vmIntrinsics::_linkToVirtual: + { + // same as TemplateTable::invokevirtual, + // minus the CP setup and profiling: + + if (VerifyMethodHandles) { + verify_ref_kind(_masm, JVM_REF_invokeVirtual, member_reg, temp3); + } + + // pick out the vtable index from the MemberName, and then we can discard it: + Register temp2_index = temp2; + __ movptr(temp2_index, member_vmindex); -void MethodHandles::move_return_value(MacroAssembler* _masm, BasicType type, - Address return_slot) { - BLOCK_COMMENT("move_return_value {"); - // Old versions of the JVM must clean the FPU stack after every return. -#ifndef _LP64 -#ifdef COMPILER2 - // The FPU stack is clean if UseSSE >= 2 but must be cleaned in other cases - if ((type == T_FLOAT && UseSSE < 1) || (type == T_DOUBLE && UseSSE < 2)) { - for (int i = 1; i < 8; i++) { - __ ffree(i); + if (VerifyMethodHandles) { + Label L_index_ok; + __ cmpl(temp2_index, 0); + __ jcc(Assembler::greaterEqual, L_index_ok); + __ STOP("no virtual index"); + __ BIND(L_index_ok); + } + + // Note: The verifier invariants allow us to ignore MemberName.clazz and vmtarget + // at this point. And VerifyMethodHandles has already checked clazz, if needed. + + // get target methodOop & entry point + __ lookup_virtual_method(temp1_recv_klass, temp2_index, rbx_method); + method_is_live = true; + break; } - } else if (UseSSE < 2) { - __ empty_FPU_stack(); - } -#endif //COMPILER2 -#endif //!_LP64 + + case vmIntrinsics::_linkToInterface: + { + // same as TemplateTable::invokeinterface + // (minus the CP setup and profiling, with different argument motion) + if (VerifyMethodHandles) { + verify_ref_kind(_masm, JVM_REF_invokeInterface, member_reg, temp3); + } + + Register temp3_intf = temp3; + __ load_heap_oop(temp3_intf, member_clazz); + load_klass_from_Class(_masm, temp3_intf); + __ verify_oop(temp3_intf); + + Register rbx_index = rbx_method; + __ movptr(rbx_index, member_vmindex); + if (VerifyMethodHandles) { + Label L; + __ cmpl(rbx_index, 0); + __ jcc(Assembler::greaterEqual, L); + __ STOP("invalid vtable index for MH.invokeInterface"); + __ bind(L); + } - // Look at the type and pull the value out of the corresponding register. - if (type == T_VOID) { - // nothing to do - } else if (type == T_OBJECT) { - __ movptr(return_slot, rax); - } else if (type == T_INT || is_subword_type(type)) { - // write the whole word, even if only 32 bits is significant - __ movptr(return_slot, rax); - } else if (type == T_LONG) { - // store the value by parts - // Note: We assume longs are continguous (if misaligned) on the interpreter stack. - __ store_sized_value(return_slot, rax, BytesPerLong, rdx); - } else if (NOT_LP64((type == T_FLOAT && UseSSE < 1) || - (type == T_DOUBLE && UseSSE < 2) ||) - false) { - // Use old x86 FPU registers: - if (type == T_FLOAT) - __ fstp_s(return_slot); - else - __ fstp_d(return_slot); - } else if (type == T_FLOAT) { - __ movflt(return_slot, xmm0); - } else if (type == T_DOUBLE) { - __ movdbl(return_slot, xmm0); - } else { - ShouldNotReachHere(); + // given intf, index, and recv klass, dispatch to the implementation method + Label L_no_such_interface; + __ lookup_interface_method(temp1_recv_klass, temp3_intf, + // note: next two args must be the same: + rbx_index, rbx_method, + temp2, + L_no_such_interface); + + __ verify_oop(rbx_method); + jump_from_method_handle(_masm, rbx_method, temp2, for_compiler_entry); + __ hlt(); + + __ bind(L_no_such_interface); + __ jump(RuntimeAddress(StubRoutines::throw_IncompatibleClassChangeError_entry())); + break; + } + + default: + fatal(err_msg("unexpected intrinsic %d: %s", iid, vmIntrinsics::name_at(iid))); + break; + } + + if (method_is_live) { + // live at this point: rbx_method, rsi/r13 (if interpreted) + + // After figuring out which concrete method to call, jump into it. + // Note that this works in the interpreter with no data motion. + // But the compiled version will require that rcx_recv be shifted out. + __ verify_oop(rbx_method); + jump_from_method_handle(_masm, rbx_method, temp1, for_compiler_entry); + } } - BLOCK_COMMENT("} move_return_value"); } #ifndef PRODUCT -#define DESCRIBE_RICOCHET_OFFSET(rf, name) \ - values.describe(frame_no, (intptr_t *) (((uintptr_t)rf) + MethodHandles::RicochetFrame::name##_offset_in_bytes()), #name) - -void MethodHandles::RicochetFrame::describe(const frame* fr, FrameValues& values, int frame_no) { - address bp = (address) fr->fp(); - RicochetFrame* rf = (RicochetFrame*)(bp - sender_link_offset_in_bytes()); - - // ricochet slots - DESCRIBE_RICOCHET_OFFSET(rf, exact_sender_sp); - DESCRIBE_RICOCHET_OFFSET(rf, conversion); - DESCRIBE_RICOCHET_OFFSET(rf, saved_args_base); - DESCRIBE_RICOCHET_OFFSET(rf, saved_args_layout); - DESCRIBE_RICOCHET_OFFSET(rf, saved_target); - DESCRIBE_RICOCHET_OFFSET(rf, continuation); - - // relevant ricochet targets (in caller frame) - values.describe(-1, rf->saved_args_base(), err_msg("*saved_args_base for #%d", frame_no)); -} -#endif // ASSERT - -#ifndef PRODUCT -extern "C" void print_method_handle(oop mh); void trace_method_handle_stub(const char* adaptername, oop mh, intptr_t* saved_regs, intptr_t* entry_sp) { // called as a leaf from native code: do not block the JVM! - bool has_mh = (strstr(adaptername, "return/") == NULL); // return adapters don't have rcx_mh + bool has_mh = (strstr(adaptername, "/static") == NULL && + strstr(adaptername, "linkTo") == NULL); // static linkers don't have MH const char* mh_reg_name = has_mh ? "rcx_mh" : "rcx"; - tty->print_cr("MH %s %s="PTR_FORMAT" sp="PTR_FORMAT, adaptername, mh_reg_name, mh, entry_sp); + tty->print_cr("MH %s %s="PTR_FORMAT" sp="PTR_FORMAT, + adaptername, mh_reg_name, + mh, entry_sp); if (Verbose) { tty->print_cr("Registers:"); @@ -1086,12 +575,18 @@ values.describe(-1, dump_fp, "fp for #1 "); values.describe(-1, dump_sp, "sp for #1"); } + values.describe(-1, entry_sp, "raw top of stack"); tty->print_cr("Stack layout:"); values.print(p); } - if (has_mh) - print_method_handle(mh); + if (has_mh && mh->is_oop()) { + mh->print(); + if (java_lang_invoke_MethodHandle::is_instance(mh)) { + if (java_lang_invoke_MethodHandle::form_offset_in_bytes() != 0) + java_lang_invoke_MethodHandle::form(mh)->print(); + } + } } } @@ -1159,1363 +654,3 @@ } #endif //PRODUCT -// which conversion op types are implemented here? -int MethodHandles::adapter_conversion_ops_supported_mask() { - return ((1<from_compiled_entry(), "method must be linked"); - - const Register rax_pc = rax; - __ pop(rax_pc); // caller PC - __ mov(rsp, saved_last_sp); // cut the stack back to where the caller started - - Register rbx_method = rbx_temp; - __ movptr(rbx_method, ExternalAddress((address) &_raise_exception_method)); - - const int jobject_oop_offset = 0; - __ movptr(rbx_method, Address(rbx_method, jobject_oop_offset)); // dereference the jobject - - __ movptr(saved_last_sp, rsp); - __ subptr(rsp, 3 * wordSize); - __ push(rax_pc); // restore caller PC - - __ movl (__ argument_address(constant(2)), rarg0_code); - __ movptr(__ argument_address(constant(1)), rarg1_actual); - __ movptr(__ argument_address(constant(0)), rarg2_required); - jump_from_method_handle(_masm, rbx_method, rax); - } - break; - - case _invokestatic_mh: - case _invokespecial_mh: - { - Register rbx_method = rbx_temp; - __ load_heap_oop(rbx_method, rcx_mh_vmtarget); // target is a methodOop - __ verify_oop(rbx_method); - // same as TemplateTable::invokestatic or invokespecial, - // minus the CP setup and profiling: - if (ek == _invokespecial_mh) { - // Must load & check the first argument before entering the target method. - __ load_method_handle_vmslots(rax_argslot, rcx_recv, rdx_temp); - __ movptr(rcx_recv, __ argument_address(rax_argslot, -1)); - __ null_check(rcx_recv); - __ verify_oop(rcx_recv); - } - jump_from_method_handle(_masm, rbx_method, rax); - } - break; - - case _invokevirtual_mh: - { - // same as TemplateTable::invokevirtual, - // minus the CP setup and profiling: - - // pick out the vtable index and receiver offset from the MH, - // and then we can discard it: - __ load_method_handle_vmslots(rax_argslot, rcx_recv, rdx_temp); - Register rbx_index = rbx_temp; - __ movl(rbx_index, rcx_dmh_vmindex); - // Note: The verifier allows us to ignore rcx_mh_vmtarget. - __ movptr(rcx_recv, __ argument_address(rax_argslot, -1)); - __ null_check(rcx_recv, oopDesc::klass_offset_in_bytes()); - - // get receiver klass - Register rax_klass = rax_argslot; - __ load_klass(rax_klass, rcx_recv); - __ verify_oop(rax_klass); - - // get target methodOop & entry point - const int base = instanceKlass::vtable_start_offset() * wordSize; - assert(vtableEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); - Address vtable_entry_addr(rax_klass, - rbx_index, Address::times_ptr, - base + vtableEntry::method_offset_in_bytes()); - Register rbx_method = rbx_temp; - __ movptr(rbx_method, vtable_entry_addr); - - __ verify_oop(rbx_method); - jump_from_method_handle(_masm, rbx_method, rax); - } - break; - - case _invokeinterface_mh: - { - // same as TemplateTable::invokeinterface, - // minus the CP setup and profiling: - - // pick out the interface and itable index from the MH. - __ load_method_handle_vmslots(rax_argslot, rcx_recv, rdx_temp); - Register rdx_intf = rdx_temp; - Register rbx_index = rbx_temp; - __ load_heap_oop(rdx_intf, rcx_mh_vmtarget); - __ movl(rbx_index, rcx_dmh_vmindex); - __ movptr(rcx_recv, __ argument_address(rax_argslot, -1)); - __ null_check(rcx_recv, oopDesc::klass_offset_in_bytes()); - - // get receiver klass - Register rax_klass = rax_argslot; - __ load_klass(rax_klass, rcx_recv); - __ verify_oop(rax_klass); - - Register rbx_method = rbx_index; - - // get interface klass - Label no_such_interface; - __ verify_oop(rdx_intf); - __ lookup_interface_method(rax_klass, rdx_intf, - // note: next two args must be the same: - rbx_index, rbx_method, - rdi_temp, - no_such_interface); - - __ verify_oop(rbx_method); - jump_from_method_handle(_masm, rbx_method, rax); - __ hlt(); - - __ bind(no_such_interface); - // Throw an exception. - // For historical reasons, it will be IncompatibleClassChangeError. - __ mov(rbx_temp, rcx_recv); // rarg2_required might be RCX - assert_different_registers(rarg2_required, rbx_temp); - __ movptr(rarg2_required, Address(rdx_intf, java_mirror_offset)); // required interface - __ mov( rarg1_actual, rbx_temp); // bad receiver - __ movl( rarg0_code, (int) Bytecodes::_invokeinterface); // who is complaining? - __ jump(ExternalAddress(from_interpreted_entry(_raise_exception))); - } - break; - - case _bound_ref_mh: - case _bound_int_mh: - case _bound_long_mh: - case _bound_ref_direct_mh: - case _bound_int_direct_mh: - case _bound_long_direct_mh: - { - const bool direct_to_method = (ek >= _bound_ref_direct_mh); - BasicType arg_type = ek_bound_mh_arg_type(ek); - int arg_slots = type2size[arg_type]; - - // make room for the new argument: - __ movl(rax_argslot, rcx_bmh_vmargslot); - __ lea(rax_argslot, __ argument_address(rax_argslot)); - - insert_arg_slots(_masm, arg_slots * stack_move_unit(), rax_argslot, rbx_temp, rdx_temp); - - // store bound argument into the new stack slot: - __ load_heap_oop(rbx_temp, rcx_bmh_argument); - if (arg_type == T_OBJECT) { - __ movptr(Address(rax_argslot, 0), rbx_temp); - } else { - Address prim_value_addr(rbx_temp, java_lang_boxing_object::value_offset_in_bytes(arg_type)); - move_typed_arg(_masm, arg_type, false, - Address(rax_argslot, 0), - prim_value_addr, - rbx_temp, rdx_temp); - } - - if (direct_to_method) { - Register rbx_method = rbx_temp; - __ load_heap_oop(rbx_method, rcx_mh_vmtarget); - __ verify_oop(rbx_method); - jump_from_method_handle(_masm, rbx_method, rax); - } else { - __ load_heap_oop(rcx_recv, rcx_mh_vmtarget); - __ verify_oop(rcx_recv); - __ jump_to_method_handle_entry(rcx_recv, rdx_temp); - } - } - break; - - case _adapter_opt_profiling: - if (java_lang_invoke_CountingMethodHandle::vmcount_offset_in_bytes() != 0) { - Address rcx_mh_vmcount(rcx_recv, java_lang_invoke_CountingMethodHandle::vmcount_offset_in_bytes()); - __ incrementl(rcx_mh_vmcount); - } - // fall through - - case _adapter_retype_only: - case _adapter_retype_raw: - // immediately jump to the next MH layer: - __ load_heap_oop(rcx_recv, rcx_mh_vmtarget); - __ verify_oop(rcx_recv); - __ jump_to_method_handle_entry(rcx_recv, rdx_temp); - // This is OK when all parameter types widen. - // It is also OK when a return type narrows. - break; - - case _adapter_check_cast: - { - // temps: - Register rbx_klass = rbx_temp; // interesting AMH data - - // check a reference argument before jumping to the next layer of MH: - __ movl(rax_argslot, rcx_amh_vmargslot); - vmarg = __ argument_address(rax_argslot); - - // What class are we casting to? - __ load_heap_oop(rbx_klass, rcx_amh_argument); // this is a Class object! - load_klass_from_Class(_masm, rbx_klass); - - Label done; - __ movptr(rdx_temp, vmarg); - __ testptr(rdx_temp, rdx_temp); - __ jcc(Assembler::zero, done); // no cast if null - __ load_klass(rdx_temp, rdx_temp); - - // live at this point: - // - rbx_klass: klass required by the target method - // - rdx_temp: argument klass to test - // - rcx_recv: adapter method handle - __ check_klass_subtype(rdx_temp, rbx_klass, rax_argslot, done); - - // If we get here, the type check failed! - // Call the wrong_method_type stub, passing the failing argument type in rax. - Register rax_mtype = rax_argslot; - __ movl(rax_argslot, rcx_amh_vmargslot); // reload argslot field - __ movptr(rdx_temp, vmarg); - - assert_different_registers(rarg2_required, rdx_temp); - __ load_heap_oop(rarg2_required, rcx_amh_argument); // required class - __ mov( rarg1_actual, rdx_temp); // bad object - __ movl( rarg0_code, (int) Bytecodes::_checkcast); // who is complaining? - __ jump(ExternalAddress(from_interpreted_entry(_raise_exception))); - - __ bind(done); - // get the new MH: - __ load_heap_oop(rcx_recv, rcx_mh_vmtarget); - __ jump_to_method_handle_entry(rcx_recv, rdx_temp); - } - break; - - case _adapter_prim_to_prim: - case _adapter_ref_to_prim: - case _adapter_prim_to_ref: - // handled completely by optimized cases - __ stop("init_AdapterMethodHandle should not issue this"); - break; - - case _adapter_opt_i2i: // optimized subcase of adapt_prim_to_prim -//case _adapter_opt_f2i: // optimized subcase of adapt_prim_to_prim - case _adapter_opt_l2i: // optimized subcase of adapt_prim_to_prim - case _adapter_opt_unboxi: // optimized subcase of adapt_ref_to_prim - { - // perform an in-place conversion to int or an int subword - __ movl(rax_argslot, rcx_amh_vmargslot); - vmarg = __ argument_address(rax_argslot); - - switch (ek) { - case _adapter_opt_i2i: - __ movl(rdx_temp, vmarg); - break; - case _adapter_opt_l2i: - { - // just delete the extra slot; on a little-endian machine we keep the first - __ lea(rax_argslot, __ argument_address(rax_argslot, 1)); - remove_arg_slots(_masm, -stack_move_unit(), - rax_argslot, rbx_temp, rdx_temp); - vmarg = Address(rax_argslot, -Interpreter::stackElementSize); - __ movl(rdx_temp, vmarg); - } - break; - case _adapter_opt_unboxi: - { - // Load the value up from the heap. - __ movptr(rdx_temp, vmarg); - int value_offset = java_lang_boxing_object::value_offset_in_bytes(T_INT); -#ifdef ASSERT - for (int bt = T_BOOLEAN; bt < T_INT; bt++) { - if (is_subword_type(BasicType(bt))) - assert(value_offset == java_lang_boxing_object::value_offset_in_bytes(BasicType(bt)), ""); - } -#endif - __ null_check(rdx_temp, value_offset); - __ movl(rdx_temp, Address(rdx_temp, value_offset)); - // We load this as a word. Because we are little-endian, - // the low bits will be correct, but the high bits may need cleaning. - // The vminfo will guide us to clean those bits. - } - break; - default: - ShouldNotReachHere(); - } - - // Do the requested conversion and store the value. - Register rbx_vminfo = rbx_temp; - load_conversion_vminfo(_masm, rbx_vminfo, rcx_amh_conversion); - - // get the new MH: - __ load_heap_oop(rcx_recv, rcx_mh_vmtarget); - // (now we are done with the old MH) - - // original 32-bit vmdata word must be of this form: - // | MBZ:6 | signBitCount:8 | srcDstTypes:8 | conversionOp:8 | - __ xchgptr(rcx, rbx_vminfo); // free rcx for shifts - __ shll(rdx_temp /*, rcx*/); - Label zero_extend, done; - __ testl(rcx, CONV_VMINFO_SIGN_FLAG); - __ jccb(Assembler::zero, zero_extend); - - // this path is taken for int->byte, int->short - __ sarl(rdx_temp /*, rcx*/); - __ jmpb(done); - - __ bind(zero_extend); - // this is taken for int->char - __ shrl(rdx_temp /*, rcx*/); - - __ bind(done); - __ movl(vmarg, rdx_temp); // Store the value. - __ xchgptr(rcx, rbx_vminfo); // restore rcx_recv - - __ jump_to_method_handle_entry(rcx_recv, rdx_temp); - } - break; - - case _adapter_opt_i2l: // optimized subcase of adapt_prim_to_prim - case _adapter_opt_unboxl: // optimized subcase of adapt_ref_to_prim - { - // perform an in-place int-to-long or ref-to-long conversion - __ movl(rax_argslot, rcx_amh_vmargslot); - - // on a little-endian machine we keep the first slot and add another after - __ lea(rax_argslot, __ argument_address(rax_argslot, 1)); - insert_arg_slots(_masm, stack_move_unit(), - rax_argslot, rbx_temp, rdx_temp); - Address vmarg1(rax_argslot, -Interpreter::stackElementSize); - Address vmarg2 = vmarg1.plus_disp(Interpreter::stackElementSize); - - switch (ek) { - case _adapter_opt_i2l: - { -#ifdef _LP64 - __ movslq(rdx_temp, vmarg1); // Load sign-extended - __ movq(vmarg1, rdx_temp); // Store into first slot -#else - __ movl(rdx_temp, vmarg1); - __ sarl(rdx_temp, BitsPerInt - 1); // __ extend_sign() - __ movl(vmarg2, rdx_temp); // store second word -#endif - } - break; - case _adapter_opt_unboxl: - { - // Load the value up from the heap. - __ movptr(rdx_temp, vmarg1); - int value_offset = java_lang_boxing_object::value_offset_in_bytes(T_LONG); - assert(value_offset == java_lang_boxing_object::value_offset_in_bytes(T_DOUBLE), ""); - __ null_check(rdx_temp, value_offset); -#ifdef _LP64 - __ movq(rbx_temp, Address(rdx_temp, value_offset)); - __ movq(vmarg1, rbx_temp); -#else - __ movl(rbx_temp, Address(rdx_temp, value_offset + 0*BytesPerInt)); - __ movl(rdx_temp, Address(rdx_temp, value_offset + 1*BytesPerInt)); - __ movl(vmarg1, rbx_temp); - __ movl(vmarg2, rdx_temp); -#endif - } - break; - default: - ShouldNotReachHere(); - } - - __ load_heap_oop(rcx_recv, rcx_mh_vmtarget); - __ jump_to_method_handle_entry(rcx_recv, rdx_temp); - } - break; - - case _adapter_opt_f2d: // optimized subcase of adapt_prim_to_prim - case _adapter_opt_d2f: // optimized subcase of adapt_prim_to_prim - { - // perform an in-place floating primitive conversion - __ movl(rax_argslot, rcx_amh_vmargslot); - __ lea(rax_argslot, __ argument_address(rax_argslot, 1)); - if (ek == _adapter_opt_f2d) { - insert_arg_slots(_masm, stack_move_unit(), - rax_argslot, rbx_temp, rdx_temp); - } - Address vmarg(rax_argslot, -Interpreter::stackElementSize); - -#ifdef _LP64 - if (ek == _adapter_opt_f2d) { - __ movflt(xmm0, vmarg); - __ cvtss2sd(xmm0, xmm0); - __ movdbl(vmarg, xmm0); - } else { - __ movdbl(xmm0, vmarg); - __ cvtsd2ss(xmm0, xmm0); - __ movflt(vmarg, xmm0); - } -#else //_LP64 - if (ek == _adapter_opt_f2d) { - __ fld_s(vmarg); // load float to ST0 - __ fstp_d(vmarg); // store double - } else { - __ fld_d(vmarg); // load double to ST0 - __ fstp_s(vmarg); // store single - } -#endif //_LP64 - - if (ek == _adapter_opt_d2f) { - remove_arg_slots(_masm, -stack_move_unit(), - rax_argslot, rbx_temp, rdx_temp); - } - - __ load_heap_oop(rcx_recv, rcx_mh_vmtarget); - __ jump_to_method_handle_entry(rcx_recv, rdx_temp); - } - break; - - case _adapter_swap_args: - case _adapter_rot_args: - // handled completely by optimized cases - __ stop("init_AdapterMethodHandle should not issue this"); - break; - - case _adapter_opt_swap_1: - case _adapter_opt_swap_2: - case _adapter_opt_rot_1_up: - case _adapter_opt_rot_1_down: - case _adapter_opt_rot_2_up: - case _adapter_opt_rot_2_down: - { - int swap_slots = ek_adapter_opt_swap_slots(ek); - int rotate = ek_adapter_opt_swap_mode(ek); - - // 'argslot' is the position of the first argument to swap - __ movl(rax_argslot, rcx_amh_vmargslot); - __ lea(rax_argslot, __ argument_address(rax_argslot)); - - // 'vminfo' is the second - Register rbx_destslot = rbx_temp; - load_conversion_vminfo(_masm, rbx_destslot, rcx_amh_conversion); - __ lea(rbx_destslot, __ argument_address(rbx_destslot)); - if (VerifyMethodHandles) - verify_argslot(_masm, rbx_destslot, "swap point must fall within current frame"); - - assert(Interpreter::stackElementSize == wordSize, "else rethink use of wordSize here"); - if (!rotate) { - // simple swap - for (int i = 0; i < swap_slots; i++) { - __ movptr(rdi_temp, Address(rax_argslot, i * wordSize)); - __ movptr(rdx_temp, Address(rbx_destslot, i * wordSize)); - __ movptr(Address(rax_argslot, i * wordSize), rdx_temp); - __ movptr(Address(rbx_destslot, i * wordSize), rdi_temp); - } - } else { - // A rotate is actually pair of moves, with an "odd slot" (or pair) - // changing place with a series of other slots. - // First, push the "odd slot", which is going to get overwritten - for (int i = swap_slots - 1; i >= 0; i--) { - // handle one with rdi_temp instead of a push: - if (i == 0) __ movptr(rdi_temp, Address(rax_argslot, i * wordSize)); - else __ pushptr( Address(rax_argslot, i * wordSize)); - } - if (rotate > 0) { - // Here is rotate > 0: - // (low mem) (high mem) - // | dest: more_slots... | arg: odd_slot :arg+1 | - // => - // | dest: odd_slot | dest+1: more_slots... :arg+1 | - // work argslot down to destslot, copying contiguous data upwards - // pseudo-code: - // rax = src_addr - swap_bytes - // rbx = dest_addr - // while (rax >= rbx) *(rax + swap_bytes) = *(rax + 0), rax--; - move_arg_slots_up(_masm, - rbx_destslot, - Address(rax_argslot, 0), - swap_slots, - rax_argslot, rdx_temp); - } else { - // Here is the other direction, rotate < 0: - // (low mem) (high mem) - // | arg: odd_slot | arg+1: more_slots... :dest+1 | - // => - // | arg: more_slots... | dest: odd_slot :dest+1 | - // work argslot up to destslot, copying contiguous data downwards - // pseudo-code: - // rax = src_addr + swap_bytes - // rbx = dest_addr - // while (rax <= rbx) *(rax - swap_bytes) = *(rax + 0), rax++; - // dest_slot denotes an exclusive upper limit - int limit_bias = OP_ROT_ARGS_DOWN_LIMIT_BIAS; - if (limit_bias != 0) - __ addptr(rbx_destslot, - limit_bias * wordSize); - move_arg_slots_down(_masm, - Address(rax_argslot, swap_slots * wordSize), - rbx_destslot, - -swap_slots, - rax_argslot, rdx_temp); - __ subptr(rbx_destslot, swap_slots * wordSize); - } - // pop the original first chunk into the destination slot, now free - for (int i = 0; i < swap_slots; i++) { - if (i == 0) __ movptr(Address(rbx_destslot, i * wordSize), rdi_temp); - else __ popptr(Address(rbx_destslot, i * wordSize)); - } - } - - __ load_heap_oop(rcx_recv, rcx_mh_vmtarget); - __ jump_to_method_handle_entry(rcx_recv, rdx_temp); - } - break; - - case _adapter_dup_args: - { - // 'argslot' is the position of the first argument to duplicate - __ movl(rax_argslot, rcx_amh_vmargslot); - __ lea(rax_argslot, __ argument_address(rax_argslot)); - - // 'stack_move' is negative number of words to duplicate - Register rdi_stack_move = rdi_temp; - load_stack_move(_masm, rdi_stack_move, rcx_recv, true); - - if (VerifyMethodHandles) { - verify_argslots(_masm, rdi_stack_move, rax_argslot, true, - "copied argument(s) must fall within current frame"); - } - - // insert location is always the bottom of the argument list: - Address insert_location = __ argument_address(constant(0)); - int pre_arg_words = insert_location.disp() / wordSize; // return PC is pushed - assert(insert_location.base() == rsp, ""); - - __ negl(rdi_stack_move); - push_arg_slots(_masm, rax_argslot, rdi_stack_move, - pre_arg_words, rbx_temp, rdx_temp); - - __ load_heap_oop(rcx_recv, rcx_mh_vmtarget); - __ jump_to_method_handle_entry(rcx_recv, rdx_temp); - } - break; - - case _adapter_drop_args: - { - // 'argslot' is the position of the first argument to nuke - __ movl(rax_argslot, rcx_amh_vmargslot); - __ lea(rax_argslot, __ argument_address(rax_argslot)); - - // (must do previous push after argslot address is taken) - - // 'stack_move' is number of words to drop - Register rdi_stack_move = rdi_temp; - load_stack_move(_masm, rdi_stack_move, rcx_recv, false); - remove_arg_slots(_masm, rdi_stack_move, - rax_argslot, rbx_temp, rdx_temp); - - __ load_heap_oop(rcx_recv, rcx_mh_vmtarget); - __ jump_to_method_handle_entry(rcx_recv, rdx_temp); - } - break; - - case _adapter_collect_args: - case _adapter_fold_args: - case _adapter_spread_args: - // handled completely by optimized cases - __ stop("init_AdapterMethodHandle should not issue this"); - break; - - case _adapter_opt_collect_ref: - case _adapter_opt_collect_int: - case _adapter_opt_collect_long: - case _adapter_opt_collect_float: - case _adapter_opt_collect_double: - case _adapter_opt_collect_void: - case _adapter_opt_collect_0_ref: - case _adapter_opt_collect_1_ref: - case _adapter_opt_collect_2_ref: - case _adapter_opt_collect_3_ref: - case _adapter_opt_collect_4_ref: - case _adapter_opt_collect_5_ref: - case _adapter_opt_filter_S0_ref: - case _adapter_opt_filter_S1_ref: - case _adapter_opt_filter_S2_ref: - case _adapter_opt_filter_S3_ref: - case _adapter_opt_filter_S4_ref: - case _adapter_opt_filter_S5_ref: - case _adapter_opt_collect_2_S0_ref: - case _adapter_opt_collect_2_S1_ref: - case _adapter_opt_collect_2_S2_ref: - case _adapter_opt_collect_2_S3_ref: - case _adapter_opt_collect_2_S4_ref: - case _adapter_opt_collect_2_S5_ref: - case _adapter_opt_fold_ref: - case _adapter_opt_fold_int: - case _adapter_opt_fold_long: - case _adapter_opt_fold_float: - case _adapter_opt_fold_double: - case _adapter_opt_fold_void: - case _adapter_opt_fold_1_ref: - case _adapter_opt_fold_2_ref: - case _adapter_opt_fold_3_ref: - case _adapter_opt_fold_4_ref: - case _adapter_opt_fold_5_ref: - { - // Given a fresh incoming stack frame, build a new ricochet frame. - // On entry, TOS points at a return PC, and RBP is the callers frame ptr. - // RSI/R13 has the caller's exact stack pointer, which we must also preserve. - // RCX contains an AdapterMethodHandle of the indicated kind. - - // Relevant AMH fields: - // amh.vmargslot: - // points to the trailing edge of the arguments - // to filter, collect, or fold. For a boxing operation, - // it points just after the single primitive value. - // amh.argument: - // recursively called MH, on |collect| arguments - // amh.vmtarget: - // final destination MH, on return value, etc. - // amh.conversion.dest: - // tells what is the type of the return value - // (not needed here, since dest is also derived from ek) - // amh.conversion.vminfo: - // points to the trailing edge of the return value - // when the vmtarget is to be called; this is - // equal to vmargslot + (retained ? |collect| : 0) - - // Pass 0 or more argument slots to the recursive target. - int collect_count_constant = ek_adapter_opt_collect_count(ek); - - // The collected arguments are copied from the saved argument list: - int collect_slot_constant = ek_adapter_opt_collect_slot(ek); - - assert(ek_orig == _adapter_collect_args || - ek_orig == _adapter_fold_args, ""); - bool retain_original_args = (ek_orig == _adapter_fold_args); - - // The return value is replaced (or inserted) at the 'vminfo' argslot. - // Sometimes we can compute this statically. - int dest_slot_constant = -1; - if (!retain_original_args) - dest_slot_constant = collect_slot_constant; - else if (collect_slot_constant >= 0 && collect_count_constant >= 0) - // We are preserving all the arguments, and the return value is prepended, - // so the return slot is to the left (above) the |collect| sequence. - dest_slot_constant = collect_slot_constant + collect_count_constant; - - // Replace all those slots by the result of the recursive call. - // The result type can be one of ref, int, long, float, double, void. - // In the case of void, nothing is pushed on the stack after return. - BasicType dest = ek_adapter_opt_collect_type(ek); - assert(dest == type2wfield[dest], "dest is a stack slot type"); - int dest_count = type2size[dest]; - assert(dest_count == 1 || dest_count == 2 || (dest_count == 0 && dest == T_VOID), "dest has a size"); - - // Choose a return continuation. - EntryKind ek_ret = _adapter_opt_return_any; - if (dest != T_CONFLICT && OptimizeMethodHandles) { - switch (dest) { - case T_INT : ek_ret = _adapter_opt_return_int; break; - case T_LONG : ek_ret = _adapter_opt_return_long; break; - case T_FLOAT : ek_ret = _adapter_opt_return_float; break; - case T_DOUBLE : ek_ret = _adapter_opt_return_double; break; - case T_OBJECT : ek_ret = _adapter_opt_return_ref; break; - case T_VOID : ek_ret = _adapter_opt_return_void; break; - default : ShouldNotReachHere(); - } - if (dest == T_OBJECT && dest_slot_constant >= 0) { - EntryKind ek_try = EntryKind(_adapter_opt_return_S0_ref + dest_slot_constant); - if (ek_try <= _adapter_opt_return_LAST && - ek_adapter_opt_return_slot(ek_try) == dest_slot_constant) { - ek_ret = ek_try; - } - } - assert(ek_adapter_opt_return_type(ek_ret) == dest, ""); - } - - // Already pushed: ... keep1 | collect | keep2 | sender_pc | - // push(sender_pc); - - // Compute argument base: - Register rax_argv = rax_argslot; - __ lea(rax_argv, __ argument_address(constant(0))); - - // Push a few extra argument words, if we need them to store the return value. - { - int extra_slots = 0; - if (retain_original_args) { - extra_slots = dest_count; - } else if (collect_count_constant == -1) { - extra_slots = dest_count; // collect_count might be zero; be generous - } else if (dest_count > collect_count_constant) { - extra_slots = (dest_count - collect_count_constant); - } else { - // else we know we have enough dead space in |collect| to repurpose for return values - } - DEBUG_ONLY(extra_slots += 1); - if (extra_slots > 0) { - __ pop(rbx_temp); // return value - __ subptr(rsp, (extra_slots * Interpreter::stackElementSize)); - // Push guard word #2 in debug mode. - DEBUG_ONLY(__ movptr(Address(rsp, 0), (int32_t) RicochetFrame::MAGIC_NUMBER_2)); - __ push(rbx_temp); - } - } - - RicochetFrame::enter_ricochet_frame(_masm, rcx_recv, rax_argv, - entry(ek_ret)->from_interpreted_entry(), rbx_temp); - - // Now pushed: ... keep1 | collect | keep2 | RF | - // some handy frame slots: - Address exact_sender_sp_addr = RicochetFrame::frame_address(RicochetFrame::exact_sender_sp_offset_in_bytes()); - Address conversion_addr = RicochetFrame::frame_address(RicochetFrame::conversion_offset_in_bytes()); - Address saved_args_base_addr = RicochetFrame::frame_address(RicochetFrame::saved_args_base_offset_in_bytes()); - -#ifdef ASSERT - if (VerifyMethodHandles && dest != T_CONFLICT) { - BLOCK_COMMENT("verify AMH.conv.dest"); - load_conversion_dest_type(_masm, rbx_temp, conversion_addr); - Label L_dest_ok; - __ cmpl(rbx_temp, (int) dest); - __ jcc(Assembler::equal, L_dest_ok); - if (dest == T_INT) { - for (int bt = T_BOOLEAN; bt < T_INT; bt++) { - if (is_subword_type(BasicType(bt))) { - __ cmpl(rbx_temp, (int) bt); - __ jcc(Assembler::equal, L_dest_ok); - } - } - } - __ stop("bad dest in AMH.conv"); - __ BIND(L_dest_ok); - } -#endif //ASSERT - - // Find out where the original copy of the recursive argument sequence begins. - Register rax_coll = rax_argv; - { - RegisterOrConstant collect_slot = collect_slot_constant; - if (collect_slot_constant == -1) { - __ movl(rdi_temp, rcx_amh_vmargslot); - collect_slot = rdi_temp; - } - if (collect_slot_constant != 0) - __ lea(rax_coll, Address(rax_argv, collect_slot, Interpreter::stackElementScale())); - // rax_coll now points at the trailing edge of |collect| and leading edge of |keep2| - } - - // Replace the old AMH with the recursive MH. (No going back now.) - // In the case of a boxing call, the recursive call is to a 'boxer' method, - // such as Integer.valueOf or Long.valueOf. In the case of a filter - // or collect call, it will take one or more arguments, transform them, - // and return some result, to store back into argument_base[vminfo]. - __ load_heap_oop(rcx_recv, rcx_amh_argument); - if (VerifyMethodHandles) verify_method_handle(_masm, rcx_recv); - - // Push a space for the recursively called MH first: - __ push((int32_t)NULL_WORD); - - // Calculate |collect|, the number of arguments we are collecting. - Register rdi_collect_count = rdi_temp; - RegisterOrConstant collect_count; - if (collect_count_constant >= 0) { - collect_count = collect_count_constant; - } else { - __ load_method_handle_vmslots(rdi_collect_count, rcx_recv, rdx_temp); - collect_count = rdi_collect_count; - } -#ifdef ASSERT - if (VerifyMethodHandles && collect_count_constant >= 0) { - __ load_method_handle_vmslots(rbx_temp, rcx_recv, rdx_temp); - Label L_count_ok; - __ cmpl(rbx_temp, collect_count_constant); - __ jcc(Assembler::equal, L_count_ok); - __ stop("bad vminfo in AMH.conv"); - __ BIND(L_count_ok); - } -#endif //ASSERT - - // copy |collect| slots directly to TOS: - push_arg_slots(_masm, rax_coll, collect_count, 0, rbx_temp, rdx_temp); - // Now pushed: ... keep1 | collect | keep2 | RF... | collect | - // rax_coll still points at the trailing edge of |collect| and leading edge of |keep2| - - // If necessary, adjust the saved arguments to make room for the eventual return value. - // Normal adjustment: ... keep1 | +dest+ | -collect- | keep2 | RF... | collect | - // If retaining args: ... keep1 | +dest+ | collect | keep2 | RF... | collect | - // In the non-retaining case, this might move keep2 either up or down. - // We don't have to copy the whole | RF... collect | complex, - // but we must adjust RF.saved_args_base. - // Also, from now on, we will forget about the original copy of |collect|. - // If we are retaining it, we will treat it as part of |keep2|. - // For clarity we will define |keep3| = |collect|keep2| or |keep2|. - - BLOCK_COMMENT("adjust trailing arguments {"); - // Compare the sizes of |+dest+| and |-collect-|, which are opposed opening and closing movements. - int open_count = dest_count; - RegisterOrConstant close_count = collect_count_constant; - Register rdi_close_count = rdi_collect_count; - if (retain_original_args) { - close_count = constant(0); - } else if (collect_count_constant == -1) { - close_count = rdi_collect_count; - } - - // How many slots need moving? This is simply dest_slot (0 => no |keep3|). - RegisterOrConstant keep3_count; - Register rsi_keep3_count = rsi; // can repair from RF.exact_sender_sp - if (dest_slot_constant >= 0) { - keep3_count = dest_slot_constant; - } else { - load_conversion_vminfo(_masm, rsi_keep3_count, conversion_addr); - keep3_count = rsi_keep3_count; - } -#ifdef ASSERT - if (VerifyMethodHandles && dest_slot_constant >= 0) { - load_conversion_vminfo(_masm, rbx_temp, conversion_addr); - Label L_vminfo_ok; - __ cmpl(rbx_temp, dest_slot_constant); - __ jcc(Assembler::equal, L_vminfo_ok); - __ stop("bad vminfo in AMH.conv"); - __ BIND(L_vminfo_ok); - } -#endif //ASSERT - - // tasks remaining: - bool move_keep3 = (!keep3_count.is_constant() || keep3_count.as_constant() != 0); - bool stomp_dest = (NOT_DEBUG(dest == T_OBJECT) DEBUG_ONLY(dest_count != 0)); - bool fix_arg_base = (!close_count.is_constant() || open_count != close_count.as_constant()); - - if (stomp_dest | fix_arg_base) { - // we will probably need an updated rax_argv value - if (collect_slot_constant >= 0) { - // rax_coll already holds the leading edge of |keep2|, so tweak it - assert(rax_coll == rax_argv, "elided a move"); - if (collect_slot_constant != 0) - __ subptr(rax_argv, collect_slot_constant * Interpreter::stackElementSize); - } else { - // Just reload from RF.saved_args_base. - __ movptr(rax_argv, saved_args_base_addr); - } - } - - // Old and new argument locations (based at slot 0). - // Net shift (&new_argv - &old_argv) is (close_count - open_count). - bool zero_open_count = (open_count == 0); // remember this bit of info - if (move_keep3 && fix_arg_base) { - // It will be easier to have everything in one register: - if (close_count.is_register()) { - // Deduct open_count from close_count register to get a clean +/- value. - __ subptr(close_count.as_register(), open_count); - } else { - close_count = close_count.as_constant() - open_count; - } - open_count = 0; - } - Address old_argv(rax_argv, 0); - Address new_argv(rax_argv, close_count, Interpreter::stackElementScale(), - - open_count * Interpreter::stackElementSize); - - // First decide if any actual data are to be moved. - // We can skip if (a) |keep3| is empty, or (b) the argument list size didn't change. - // (As it happens, all movements involve an argument list size change.) - - // If there are variable parameters, use dynamic checks to skip around the whole mess. - Label L_done; - if (!keep3_count.is_constant()) { - __ testl(keep3_count.as_register(), keep3_count.as_register()); - __ jcc(Assembler::zero, L_done); - } - if (!close_count.is_constant()) { - __ cmpl(close_count.as_register(), open_count); - __ jcc(Assembler::equal, L_done); - } - - if (move_keep3 && fix_arg_base) { - bool emit_move_down = false, emit_move_up = false, emit_guard = false; - if (!close_count.is_constant()) { - emit_move_down = emit_guard = !zero_open_count; - emit_move_up = true; - } else if (open_count != close_count.as_constant()) { - emit_move_down = (open_count > close_count.as_constant()); - emit_move_up = !emit_move_down; - } - Label L_move_up; - if (emit_guard) { - __ cmpl(close_count.as_register(), open_count); - __ jcc(Assembler::greater, L_move_up); - } - - if (emit_move_down) { - // Move arguments down if |+dest+| > |-collect-| - // (This is rare, except when arguments are retained.) - // This opens space for the return value. - if (keep3_count.is_constant()) { - for (int i = 0; i < keep3_count.as_constant(); i++) { - __ movptr(rdx_temp, old_argv.plus_disp(i * Interpreter::stackElementSize)); - __ movptr( new_argv.plus_disp(i * Interpreter::stackElementSize), rdx_temp); - } - } else { - Register rbx_argv_top = rbx_temp; - __ lea(rbx_argv_top, old_argv.plus_disp(keep3_count, Interpreter::stackElementScale())); - move_arg_slots_down(_masm, - old_argv, // beginning of old argv - rbx_argv_top, // end of old argv - close_count, // distance to move down (must be negative) - rax_argv, rdx_temp); - // Used argv as an iteration variable; reload from RF.saved_args_base. - __ movptr(rax_argv, saved_args_base_addr); - } - } - - if (emit_guard) { - __ jmp(L_done); // assumes emit_move_up is true also - __ BIND(L_move_up); - } - - if (emit_move_up) { - - // Move arguments up if |+dest+| < |-collect-| - // (This is usual, except when |keep3| is empty.) - // This closes up the space occupied by the now-deleted collect values. - if (keep3_count.is_constant()) { - for (int i = keep3_count.as_constant() - 1; i >= 0; i--) { - __ movptr(rdx_temp, old_argv.plus_disp(i * Interpreter::stackElementSize)); - __ movptr( new_argv.plus_disp(i * Interpreter::stackElementSize), rdx_temp); - } - } else { - Address argv_top = old_argv.plus_disp(keep3_count, Interpreter::stackElementScale()); - move_arg_slots_up(_masm, - rax_argv, // beginning of old argv - argv_top, // end of old argv - close_count, // distance to move up (must be positive) - rbx_temp, rdx_temp); - } - } - } - __ BIND(L_done); - - if (fix_arg_base) { - // adjust RF.saved_args_base by adding (close_count - open_count) - if (!new_argv.is_same_address(Address(rax_argv, 0))) - __ lea(rax_argv, new_argv); - __ movptr(saved_args_base_addr, rax_argv); - } - - if (stomp_dest) { - // Stomp the return slot, so it doesn't hold garbage. - // This isn't strictly necessary, but it may help detect bugs. - int forty_two = RicochetFrame::RETURN_VALUE_PLACEHOLDER; - __ movptr(Address(rax_argv, keep3_count, Address::times_ptr), - (int32_t) forty_two); - // uses rsi_keep3_count - } - BLOCK_COMMENT("} adjust trailing arguments"); - - BLOCK_COMMENT("do_recursive_call"); - __ mov(saved_last_sp, rsp); // set rsi/r13 for callee - __ pushptr(ExternalAddress(SharedRuntime::ricochet_blob()->bounce_addr()).addr()); - // The globally unique bounce address has two purposes: - // 1. It helps the JVM recognize this frame (frame::is_ricochet_frame). - // 2. When returned to, it cuts back the stack and redirects control flow - // to the return handler. - // The return handler will further cut back the stack when it takes - // down the RF. Perhaps there is a way to streamline this further. - - // State during recursive call: - // ... keep1 | dest | dest=42 | keep3 | RF... | collect | bounce_pc | - __ jump_to_method_handle_entry(rcx_recv, rdx_temp); - - break; - } - - case _adapter_opt_return_ref: - case _adapter_opt_return_int: - case _adapter_opt_return_long: - case _adapter_opt_return_float: - case _adapter_opt_return_double: - case _adapter_opt_return_void: - case _adapter_opt_return_S0_ref: - case _adapter_opt_return_S1_ref: - case _adapter_opt_return_S2_ref: - case _adapter_opt_return_S3_ref: - case _adapter_opt_return_S4_ref: - case _adapter_opt_return_S5_ref: - { - BasicType dest_type_constant = ek_adapter_opt_return_type(ek); - int dest_slot_constant = ek_adapter_opt_return_slot(ek); - - if (VerifyMethodHandles) RicochetFrame::verify_clean(_masm); - - if (dest_slot_constant == -1) { - // The current stub is a general handler for this dest_type. - // It can be called from _adapter_opt_return_any below. - // Stash the address in a little table. - assert((dest_type_constant & CONV_TYPE_MASK) == dest_type_constant, "oob"); - address return_handler = __ pc(); - _adapter_return_handlers[dest_type_constant] = return_handler; - if (dest_type_constant == T_INT) { - // do the subword types too - for (int bt = T_BOOLEAN; bt < T_INT; bt++) { - if (is_subword_type(BasicType(bt)) && - _adapter_return_handlers[bt] == NULL) { - _adapter_return_handlers[bt] = return_handler; - } - } - } - } - - Register rbx_arg_base = rbx_temp; - assert_different_registers(rax, rdx, // possibly live return value registers - rdi_temp, rbx_arg_base); - - Address conversion_addr = RicochetFrame::frame_address(RicochetFrame::conversion_offset_in_bytes()); - Address saved_args_base_addr = RicochetFrame::frame_address(RicochetFrame::saved_args_base_offset_in_bytes()); - - __ movptr(rbx_arg_base, saved_args_base_addr); - RegisterOrConstant dest_slot = dest_slot_constant; - if (dest_slot_constant == -1) { - load_conversion_vminfo(_masm, rdi_temp, conversion_addr); - dest_slot = rdi_temp; - } - // Store the result back into the argslot. - // This code uses the interpreter calling sequence, in which the return value - // is usually left in the TOS register, as defined by InterpreterMacroAssembler::pop. - // There are certain irregularities with floating point values, which can be seen - // in TemplateInterpreterGenerator::generate_return_entry_for. - move_return_value(_masm, dest_type_constant, Address(rbx_arg_base, dest_slot, Interpreter::stackElementScale())); - - RicochetFrame::leave_ricochet_frame(_masm, rcx_recv, rbx_arg_base, rdx_temp); - __ push(rdx_temp); // repush the return PC - - // Load the final target and go. - if (VerifyMethodHandles) verify_method_handle(_masm, rcx_recv); - __ jump_to_method_handle_entry(rcx_recv, rdx_temp); - __ hlt(); // -------------------- - break; - } - - case _adapter_opt_return_any: - { - if (VerifyMethodHandles) RicochetFrame::verify_clean(_masm); - Register rdi_conv = rdi_temp; - assert_different_registers(rax, rdx, // possibly live return value registers - rdi_conv, rbx_temp); - - Address conversion_addr = RicochetFrame::frame_address(RicochetFrame::conversion_offset_in_bytes()); - load_conversion_dest_type(_masm, rdi_conv, conversion_addr); - __ lea(rbx_temp, ExternalAddress((address) &_adapter_return_handlers[0])); - __ movptr(rbx_temp, Address(rbx_temp, rdi_conv, Address::times_ptr)); - -#ifdef ASSERT - { Label L_badconv; - __ testptr(rbx_temp, rbx_temp); - __ jccb(Assembler::zero, L_badconv); - __ jmp(rbx_temp); - __ bind(L_badconv); - __ stop("bad method handle return"); - } -#else //ASSERT - __ jmp(rbx_temp); -#endif //ASSERT - break; - } - - case _adapter_opt_spread_0: - case _adapter_opt_spread_1_ref: - case _adapter_opt_spread_2_ref: - case _adapter_opt_spread_3_ref: - case _adapter_opt_spread_4_ref: - case _adapter_opt_spread_5_ref: - case _adapter_opt_spread_ref: - case _adapter_opt_spread_byte: - case _adapter_opt_spread_char: - case _adapter_opt_spread_short: - case _adapter_opt_spread_int: - case _adapter_opt_spread_long: - case _adapter_opt_spread_float: - case _adapter_opt_spread_double: - { - // spread an array out into a group of arguments - int length_constant = ek_adapter_opt_spread_count(ek); - bool length_can_be_zero = (length_constant == 0); - if (length_constant < 0) { - // some adapters with variable length must handle the zero case - if (!OptimizeMethodHandles || - ek_adapter_opt_spread_type(ek) != T_OBJECT) - length_can_be_zero = true; - } - - // find the address of the array argument - __ movl(rax_argslot, rcx_amh_vmargslot); - __ lea(rax_argslot, __ argument_address(rax_argslot)); - - // grab another temp - Register rsi_temp = rsi; - - // arx_argslot points both to the array and to the first output arg - vmarg = Address(rax_argslot, 0); - - // Get the array value. - Register rdi_array = rdi_temp; - Register rdx_array_klass = rdx_temp; - BasicType elem_type = ek_adapter_opt_spread_type(ek); - int elem_slots = type2size[elem_type]; // 1 or 2 - int array_slots = 1; // array is always a T_OBJECT - int length_offset = arrayOopDesc::length_offset_in_bytes(); - int elem0_offset = arrayOopDesc::base_offset_in_bytes(elem_type); - __ movptr(rdi_array, vmarg); - - Label L_array_is_empty, L_insert_arg_space, L_copy_args, L_args_done; - if (length_can_be_zero) { - // handle the null pointer case, if zero is allowed - Label L_skip; - if (length_constant < 0) { - load_conversion_vminfo(_masm, rbx_temp, rcx_amh_conversion); - __ testl(rbx_temp, rbx_temp); - __ jcc(Assembler::notZero, L_skip); - } - __ testptr(rdi_array, rdi_array); - __ jcc(Assembler::notZero, L_skip); - - // If 'rsi' contains the 'saved_last_sp' (this is only the - // case in a 32-bit version of the VM) we have to save 'rsi' - // on the stack because later on (at 'L_array_is_empty') 'rsi' - // will be overwritten. - { if (rsi_temp == saved_last_sp) __ push(saved_last_sp); } - // Also prepare a handy macro which restores 'rsi' if required. -#define UNPUSH_RSI \ - { if (rsi_temp == saved_last_sp) __ pop(saved_last_sp); } - - __ jmp(L_array_is_empty); - __ bind(L_skip); - } - __ null_check(rdi_array, oopDesc::klass_offset_in_bytes()); - __ load_klass(rdx_array_klass, rdi_array); - - // Save 'rsi' if required (see comment above). Do this only - // after the null check such that the exception handler which is - // called in the case of a null pointer exception will not be - // confused by the extra value on the stack (it expects the - // return pointer on top of the stack) - { if (rsi_temp == saved_last_sp) __ push(saved_last_sp); } - - // Check the array type. - Register rbx_klass = rbx_temp; - __ load_heap_oop(rbx_klass, rcx_amh_argument); // this is a Class object! - load_klass_from_Class(_masm, rbx_klass); - - Label ok_array_klass, bad_array_klass, bad_array_length; - __ check_klass_subtype(rdx_array_klass, rbx_klass, rsi_temp, ok_array_klass); - // If we get here, the type check failed! - __ jmp(bad_array_klass); - __ BIND(ok_array_klass); - - // Check length. - if (length_constant >= 0) { - __ cmpl(Address(rdi_array, length_offset), length_constant); - } else { - Register rbx_vminfo = rbx_temp; - load_conversion_vminfo(_masm, rbx_vminfo, rcx_amh_conversion); - __ cmpl(rbx_vminfo, Address(rdi_array, length_offset)); - } - __ jcc(Assembler::notEqual, bad_array_length); - - Register rdx_argslot_limit = rdx_temp; - - // Array length checks out. Now insert any required stack slots. - if (length_constant == -1) { - // Form a pointer to the end of the affected region. - __ lea(rdx_argslot_limit, Address(rax_argslot, Interpreter::stackElementSize)); - // 'stack_move' is negative number of words to insert - // This number already accounts for elem_slots. - Register rsi_stack_move = rsi_temp; - load_stack_move(_masm, rsi_stack_move, rcx_recv, true); - __ cmpptr(rsi_stack_move, 0); - assert(stack_move_unit() < 0, "else change this comparison"); - __ jcc(Assembler::less, L_insert_arg_space); - __ jcc(Assembler::equal, L_copy_args); - // single argument case, with no array movement - __ BIND(L_array_is_empty); - remove_arg_slots(_masm, -stack_move_unit() * array_slots, - rax_argslot, rbx_temp, rdx_temp); - __ jmp(L_args_done); // no spreading to do - __ BIND(L_insert_arg_space); - // come here in the usual case, stack_move < 0 (2 or more spread arguments) - Register rdi_temp = rdi_array; // spill this - insert_arg_slots(_masm, rsi_stack_move, - rax_argslot, rbx_temp, rdi_temp); - // reload the array since rsi was killed - // reload from rdx_argslot_limit since rax_argslot is now decremented - __ movptr(rdi_array, Address(rdx_argslot_limit, -Interpreter::stackElementSize)); - } else if (length_constant >= 1) { - int new_slots = (length_constant * elem_slots) - array_slots; - insert_arg_slots(_masm, new_slots * stack_move_unit(), - rax_argslot, rbx_temp, rdx_temp); - } else if (length_constant == 0) { - __ BIND(L_array_is_empty); - remove_arg_slots(_masm, -stack_move_unit() * array_slots, - rax_argslot, rbx_temp, rdx_temp); - } else { - ShouldNotReachHere(); - } - - // Copy from the array to the new slots. - // Note: Stack change code preserves integrity of rax_argslot pointer. - // So even after slot insertions, rax_argslot still points to first argument. - // Beware: Arguments that are shallow on the stack are deep in the array, - // and vice versa. So a downward-growing stack (the usual) has to be copied - // elementwise in reverse order from the source array. - __ BIND(L_copy_args); - if (length_constant == -1) { - // [rax_argslot, rdx_argslot_limit) is the area we are inserting into. - // Array element [0] goes at rdx_argslot_limit[-wordSize]. - Register rdi_source = rdi_array; - __ lea(rdi_source, Address(rdi_array, elem0_offset)); - Register rdx_fill_ptr = rdx_argslot_limit; - Label loop; - __ BIND(loop); - __ addptr(rdx_fill_ptr, -Interpreter::stackElementSize * elem_slots); - move_typed_arg(_masm, elem_type, true, - Address(rdx_fill_ptr, 0), Address(rdi_source, 0), - rbx_temp, rsi_temp); - __ addptr(rdi_source, type2aelembytes(elem_type)); - __ cmpptr(rdx_fill_ptr, rax_argslot); - __ jcc(Assembler::above, loop); - } else if (length_constant == 0) { - // nothing to copy - } else { - int elem_offset = elem0_offset; - int slot_offset = length_constant * Interpreter::stackElementSize; - for (int index = 0; index < length_constant; index++) { - slot_offset -= Interpreter::stackElementSize * elem_slots; // fill backward - move_typed_arg(_masm, elem_type, true, - Address(rax_argslot, slot_offset), Address(rdi_array, elem_offset), - rbx_temp, rsi_temp); - elem_offset += type2aelembytes(elem_type); - } - } - __ BIND(L_args_done); - - // Arguments are spread. Move to next method handle. - UNPUSH_RSI; - __ load_heap_oop(rcx_recv, rcx_mh_vmtarget); - __ jump_to_method_handle_entry(rcx_recv, rdx_temp); - - __ bind(bad_array_klass); - UNPUSH_RSI; - assert(!vmarg.uses(rarg2_required), "must be different registers"); - __ load_heap_oop( rarg2_required, Address(rdx_array_klass, java_mirror_offset)); // required type - __ movptr( rarg1_actual, vmarg); // bad array - __ movl( rarg0_code, (int) Bytecodes::_aaload); // who is complaining? - __ jump(ExternalAddress(from_interpreted_entry(_raise_exception))); - - __ bind(bad_array_length); - UNPUSH_RSI; - assert(!vmarg.uses(rarg2_required), "must be different registers"); - __ mov( rarg2_required, rcx_recv); // AMH requiring a certain length - __ movptr( rarg1_actual, vmarg); // bad array - __ movl( rarg0_code, (int) Bytecodes::_arraylength); // who is complaining? - __ jump(ExternalAddress(from_interpreted_entry(_raise_exception))); -#undef UNPUSH_RSI - - break; - } - - default: - // do not require all platforms to recognize all adapter types - __ nop(); - return; - } - BLOCK_COMMENT(err_msg("} Entry %s", entry_name(ek))); - __ hlt(); - - address me_cookie = MethodHandleEntry::start_compiled_entry(_masm, interp_entry); - __ unimplemented(entry_name(ek)); // %%% FIXME: NYI - - init_entry(ek, MethodHandleEntry::finish_compiled_entry(_masm, me_cookie)); -}