# HG changeset patch # User trims # Date 1296615473 28800 # Node ID 5eecea832c836f0d910191740b7522f104890de6 # Parent 25b4fa8c7dd24296713e62cd1e3998cdfea9e666# Parent c2f2ee35ff819ef929dee523262e98902871a4e9 Merge diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/.hgtags --- a/hotspot/.hgtags Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/.hgtags Tue Feb 01 18:57:53 2011 -0800 @@ -142,4 +142,5 @@ 0a8e0d4345b37b71ec49dda08ee03b68c4f1b592 hs20-b05 e24ab3fa6aafad3efabbe7dba9918c5f461a20b1 jdk7-b125 4c851c931d001a882cab809aaf3a55371b919244 jdk7-b126 +e24ab3fa6aafad3efabbe7dba9918c5f461a20b1 hs20-b06 102466e70debc4b907afbd7624e34ddb1aafee9f jdk7-b127 diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/make/hotspot_version --- a/hotspot/make/hotspot_version Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/make/hotspot_version Tue Feb 01 18:57:53 2011 -0800 @@ -35,7 +35,7 @@ HS_MAJOR_VER=20 HS_MINOR_VER=0 -HS_BUILD_NUMBER=06 +HS_BUILD_NUMBER=07 JDK_MAJOR_VER=1 JDK_MINOR_VER=7 diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/sparc/vm/assembler_sparc.cpp --- a/hotspot/src/cpu/sparc/vm/assembler_sparc.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/assembler_sparc.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -4104,7 +4104,7 @@ ld_ptr(G2_thread, in_bytes(JavaThread::tlab_start_offset()), t1); sub(top, t1, t1); // size of tlab's allocated portion - incr_allocated_bytes(t1, 0, t2); + incr_allocated_bytes(t1, t2, t3); // refill the tlab with an eden allocation bind(do_refill); @@ -4138,19 +4138,14 @@ delayed()->nop(); } -void MacroAssembler::incr_allocated_bytes(Register var_size_in_bytes, - int con_size_in_bytes, - Register t1) { +void MacroAssembler::incr_allocated_bytes(RegisterOrConstant size_in_bytes, + Register t1, Register t2) { // Bump total bytes allocated by this thread assert(t1->is_global(), "must be global reg"); // so all 64 bits are saved on a context switch - assert_different_registers(var_size_in_bytes, t1); + assert_different_registers(size_in_bytes.register_or_noreg(), t1, t2); // v8 support has gone the way of the dodo ldx(G2_thread, in_bytes(JavaThread::allocated_bytes_offset()), t1); - if (var_size_in_bytes->is_valid()) { - add(t1, var_size_in_bytes, t1); - } else { - add(t1, con_size_in_bytes, t1); - } + add(t1, ensure_simm13_or_reg(size_in_bytes, t2), t1); stx(t1, G2_thread, in_bytes(JavaThread::allocated_bytes_offset())); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/sparc/vm/assembler_sparc.hpp --- a/hotspot/src/cpu/sparc/vm/assembler_sparc.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/assembler_sparc.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -823,15 +823,23 @@ }; // test if x is within signed immediate range for nbits - static bool is_simm(int x, int nbits) { return -( 1 << nbits-1 ) <= x && x < ( 1 << nbits-1 ); } + static bool is_simm(intptr_t x, int nbits) { return -( intptr_t(1) << nbits-1 ) <= x && x < ( intptr_t(1) << nbits-1 ); } // test if -4096 <= x <= 4095 - static bool is_simm13(int x) { return is_simm(x, 13); } + static bool is_simm13(intptr_t x) { return is_simm(x, 13); } + + static bool is_in_wdisp_range(address a, address b, int nbits) { + intptr_t d = intptr_t(b) - intptr_t(a); + return is_simm(d, nbits + 2); + } // test if label is in simm16 range in words (wdisp16). bool is_in_wdisp16_range(Label& L) { - intptr_t d = intptr_t(pc()) - intptr_t(target(L)); - return is_simm(d, 18); + return is_in_wdisp_range(target(L), pc(), 16); + } + // test if the distance between two addresses fits in simm30 range in words + static bool is_in_wdisp30_range(address a, address b) { + return is_in_wdisp_range(a, b, 30); } enum ASIs { // page 72, v9 @@ -1843,6 +1851,8 @@ inline void jmp( Register s1, Register s2 ); inline void jmp( Register s1, int simm13a, RelocationHolder const& rspec = RelocationHolder() ); + // Check if the call target is out of wdisp30 range (relative to the code cache) + static inline bool is_far_target(address d); inline void call( address d, relocInfo::relocType rt = relocInfo::runtime_call_type ); inline void call( Label& L, relocInfo::relocType rt = relocInfo::runtime_call_type ); inline void callr( Register s1, Register s2 ); @@ -2389,7 +2399,8 @@ Label& slow_case // continuation point if fast allocation fails ); void tlab_refill(Label& retry_tlab, Label& try_eden, Label& slow_case); - void incr_allocated_bytes(Register var_size_in_bytes, int con_size_in_bytes, Register t1); + void incr_allocated_bytes(RegisterOrConstant size_in_bytes, + Register t1, Register t2); // interface method calling void lookup_interface_method(Register recv_klass, diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/sparc/vm/assembler_sparc.inline.hpp --- a/hotspot/src/cpu/sparc/vm/assembler_sparc.inline.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/assembler_sparc.inline.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -588,10 +588,13 @@ inline void MacroAssembler::jmp( Register s1, Register s2 ) { jmpl( s1, s2, G0 ); } inline void MacroAssembler::jmp( Register s1, int simm13a, RelocationHolder const& rspec ) { jmpl( s1, simm13a, G0, rspec); } +inline bool MacroAssembler::is_far_target(address d) { + return !is_in_wdisp30_range(d, CodeCache::low_bound()) || !is_in_wdisp30_range(d, CodeCache::high_bound()); +} + // Call with a check to see if we need to deal with the added // expense of relocation and if we overflow the displacement -// of the quick call instruction./ -// Check to see if we have to deal with relocations +// of the quick call instruction. inline void MacroAssembler::call( address d, relocInfo::relocType rt ) { #ifdef _LP64 intptr_t disp; @@ -603,14 +606,12 @@ // Is this address within range of the call instruction? // If not, use the expensive instruction sequence - disp = (intptr_t)d - (intptr_t)pc(); - if ( disp != (intptr_t)(int32_t)disp ) { + if (is_far_target(d)) { relocate(rt); AddressLiteral dest(d); jumpl_to(dest, O7, O7); - } - else { - Assembler::call( d, rt ); + } else { + Assembler::call(d, rt); } #else Assembler::call( d, rt ); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/sparc/vm/c1_LIRAssembler_sparc.cpp --- a/hotspot/src/cpu/sparc/vm/c1_LIRAssembler_sparc.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/c1_LIRAssembler_sparc.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -2358,6 +2358,8 @@ op->tmp3()->as_register() == G4 && op->tmp4()->as_register() == O1 && op->klass()->as_register() == G5, "must be"); + + LP64_ONLY( __ signx(op->len()->as_register()); ) if (UseSlowPath || (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) || (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) { diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/sparc/vm/c1_MacroAssembler_sparc.cpp --- a/hotspot/src/cpu/sparc/vm/c1_MacroAssembler_sparc.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/c1_MacroAssembler_sparc.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -170,11 +170,13 @@ Register t2, // temp register Label& slow_case // continuation point if fast allocation fails ) { + RegisterOrConstant size_in_bytes = var_size_in_bytes->is_valid() + ? RegisterOrConstant(var_size_in_bytes) : RegisterOrConstant(con_size_in_bytes); if (UseTLAB) { tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case); } else { eden_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); - incr_allocated_bytes(var_size_in_bytes, con_size_in_bytes, t1); + incr_allocated_bytes(size_in_bytes, t1, t2); } } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/sparc/vm/c1_Runtime1_sparc.cpp --- a/hotspot/src/cpu/sparc/vm/c1_Runtime1_sparc.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/c1_Runtime1_sparc.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -343,9 +343,10 @@ // returned. restore_live_registers(sasm); - __ restore(); - __ br(Assembler::always, false, Assembler::pt, deopt_blob->unpack_with_reexecution(), relocInfo::runtime_call_type); - __ delayed()->nop(); + + AddressLiteral dest(deopt_blob->unpack_with_reexecution()); + __ jump_to(dest, O0); + __ delayed()->restore(); __ bind(no_deopt); restore_live_registers(sasm); @@ -461,7 +462,7 @@ // get the instance size __ ld(G5_klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes(), G1_obj_size); __ eden_allocate(O0_obj, G1_obj_size, 0, G3_t1, G4_t2, slow_path); - __ incr_allocated_bytes(G1_obj_size, 0, G3_t1); + __ incr_allocated_bytes(G1_obj_size, G3_t1, G4_t2); __ initialize_object(O0_obj, G5_klass, G1_obj_size, 0, G3_t1, G4_t2); __ verify_oop(O0_obj); @@ -577,7 +578,7 @@ __ and3(G1_arr_size, ~MinObjAlignmentInBytesMask, G1_arr_size); __ eden_allocate(O0_obj, G1_arr_size, 0, G3_t1, O1_t2, slow_path); // preserves G1_arr_size - __ incr_allocated_bytes(G1_arr_size, 0, G3_t1); + __ incr_allocated_bytes(G1_arr_size, G3_t1, O1_t2); __ initialize_header(O0_obj, G5_klass, G4_length, G3_t1, O1_t2); __ ldub(klass_lh, G3_t1, klass_lh_header_size_offset); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/sparc/vm/interp_masm_sparc.cpp --- a/hotspot/src/cpu/sparc/vm/interp_masm_sparc.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/interp_masm_sparc.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1295,16 +1295,13 @@ // Get the method data pointer from the methodOop and set the // specified register to its value. -void InterpreterMacroAssembler::set_method_data_pointer_offset(Register Roff) { +void InterpreterMacroAssembler::set_method_data_pointer() { assert(ProfileInterpreter, "must be profiling interpreter"); Label get_continue; ld_ptr(Lmethod, in_bytes(methodOopDesc::method_data_offset()), ImethodDataPtr); test_method_data_pointer(get_continue); add(ImethodDataPtr, in_bytes(methodDataOopDesc::data_offset()), ImethodDataPtr); - if (Roff != noreg) - // Roff contains a method data index ("mdi"). It defaults to zero. - add(ImethodDataPtr, Roff, ImethodDataPtr); bind(get_continue); } @@ -1315,10 +1312,11 @@ Label zero_continue; // Test MDO to avoid the call if it is NULL. - ld_ptr(Lmethod, methodOopDesc::method_data_offset(), ImethodDataPtr); + ld_ptr(Lmethod, in_bytes(methodOopDesc::method_data_offset()), ImethodDataPtr); test_method_data_pointer(zero_continue); call_VM_leaf(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), Lmethod, Lbcp); - set_method_data_pointer_offset(O0); + add(ImethodDataPtr, in_bytes(methodDataOopDesc::data_offset()), ImethodDataPtr); + add(ImethodDataPtr, O0, ImethodDataPtr); bind(zero_continue); } @@ -1369,7 +1367,6 @@ } void InterpreterMacroAssembler::test_invocation_counter_for_mdp(Register invocation_count, - Register cur_bcp, Register Rtmp, Label &profile_continue) { assert(ProfileInterpreter, "must be profiling interpreter"); @@ -1400,8 +1397,8 @@ delayed()->nop(); // Build it now. - call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method), cur_bcp); - set_method_data_pointer_offset(O0); + call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method)); + set_method_data_pointer_for_bcp(); ba(false, profile_continue); delayed()->nop(); bind(done); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/sparc/vm/interp_masm_sparc.hpp --- a/hotspot/src/cpu/sparc/vm/interp_masm_sparc.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/interp_masm_sparc.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -269,12 +269,11 @@ #ifndef CC_INTERP // Interpreter profiling operations - void set_method_data_pointer() { set_method_data_pointer_offset(noreg); } + void set_method_data_pointer(); void set_method_data_pointer_for_bcp(); - void set_method_data_pointer_offset(Register mdi_reg); void test_method_data_pointer(Label& zero_continue); void verify_method_data_pointer(); - void test_invocation_counter_for_mdp(Register invocation_count, Register cur_bcp, Register Rtmp, Label &profile_continue); + void test_invocation_counter_for_mdp(Register invocation_count, Register Rtmp, Label &profile_continue); void set_mdp_data_at(int constant, Register value); void increment_mdp_data_at(Address counter, Register bumped_count, diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/sparc/vm/methodHandles_sparc.cpp --- a/hotspot/src/cpu/sparc/vm/methodHandles_sparc.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/methodHandles_sparc.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -395,7 +395,7 @@ // // Generate an "entry" field for a method handle. // This determines how the method handle will respond to calls. -void MethodHandles::generate_method_handle_stub(MacroAssembler* _masm, MethodHandles::EntryKind ek, TRAPS) { +void MethodHandles::generate_method_handle_stub(MacroAssembler* _masm, MethodHandles::EntryKind ek) { // Here is the register state during an interpreted call, // as set up by generate_method_handle_interpreter_entry(): // - G5: garbage temp (was MethodHandle.invoke methodOop, unused) @@ -447,8 +447,9 @@ // exception. Since we use a C2I adapter to set up the // interpreter state, arguments are expected in compiler // argument registers. - methodHandle mh(raise_exception_method()); - address c2i_entry = methodOopDesc::make_adapters(mh, CATCH); + assert(raise_exception_method(), "must be set"); + address c2i_entry = raise_exception_method()->get_c2i_entry(); + assert(c2i_entry, "method must be linked"); __ mov(O5_savedSP, SP); // Cut the stack back to where the caller started. diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/sparc/vm/sparc.ad --- a/hotspot/src/cpu/sparc/vm/sparc.ad Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/sparc.ad Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ // -// Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 1998, 2011, Oracle and/or its affiliates. All rights reserved. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -575,7 +575,11 @@ int MachCallRuntimeNode::ret_addr_offset() { #ifdef _LP64 - return NativeFarCall::instruction_size; // farcall; delay slot + if (MacroAssembler::is_far_target(entry_point())) { + return NativeFarCall::instruction_size; + } else { + return NativeCall::instruction_size; + } #else return NativeCall::instruction_size; // call; delay slot #endif @@ -941,7 +945,7 @@ #endif } -void emit_call_reloc(CodeBuffer &cbuf, intptr_t entry_point, relocInfo::relocType rtype, bool preserve_g2 = false, bool force_far_call = false) { +void emit_call_reloc(CodeBuffer &cbuf, intptr_t entry_point, relocInfo::relocType rtype, bool preserve_g2 = false) { // The method which records debug information at every safepoint // expects the call to be the first instruction in the snippet as // it creates a PcDesc structure which tracks the offset of a call @@ -963,20 +967,7 @@ int startpos = __ offset(); #endif /* ASSERT */ -#ifdef _LP64 - // Calls to the runtime or native may not be reachable from compiled code, - // so we generate the far call sequence on 64 bit sparc. - // This code sequence is relocatable to any address, even on LP64. - if ( force_far_call ) { - __ relocate(rtype); - AddressLiteral dest(entry_point); - __ jumpl_to(dest, O7, O7); - } - else -#endif - { - __ call((address)entry_point, rtype); - } + __ call((address)entry_point, rtype); if (preserve_g2) __ delayed()->mov(G2, L7); else __ delayed()->nop(); @@ -2507,7 +2498,7 @@ // CALL directly to the runtime // The user of this is responsible for ensuring that R_L7 is empty (killed). emit_call_reloc(cbuf, $meth$$method, relocInfo::runtime_call_type, - /*preserve_g2=*/true, /*force far call*/true); + /*preserve_g2=*/true); %} enc_class preserve_SP %{ diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/sparc/vm/templateInterpreter_sparc.cpp --- a/hotspot/src/cpu/sparc/vm/templateInterpreter_sparc.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/templateInterpreter_sparc.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1364,15 +1364,8 @@ // We have decided to profile this method in the interpreter __ bind(profile_method); - __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method), Lbcp, true); - -#ifdef ASSERT - __ tst(O0); - __ breakpoint_trap(Assembler::notEqual); -#endif - - __ set_method_data_pointer(); - + __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method)); + __ set_method_data_pointer_for_bcp(); __ ba(false, profile_method_continue); __ delayed()->nop(); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/sparc/vm/templateTable_sparc.cpp --- a/hotspot/src/cpu/sparc/vm/templateTable_sparc.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/sparc/vm/templateTable_sparc.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1689,7 +1689,7 @@ const Register G4_invoke_ctr = G4; __ increment_backedge_counter(G4_invoke_ctr, G1_scratch); if (ProfileInterpreter) { - __ test_invocation_counter_for_mdp(G4_invoke_ctr, Lbcp, G3_scratch, Lforward); + __ test_invocation_counter_for_mdp(G4_invoke_ctr, G3_scratch, Lforward); if (UseOnStackReplacement) { __ test_backedge_count_for_osr(O2_bumped_count, O0_cur_bcp, G3_scratch); } @@ -3447,7 +3447,8 @@ __ delayed()->nop(); // bump total bytes allocated by this thread - __ incr_allocated_bytes(Roffset, 0, G1_scratch); + // RoldTopValue and RtopAddr are dead, so can use G1 and G3 + __ incr_allocated_bytes(Roffset, G1_scratch, G3_scratch); } if (UseTLAB || Universe::heap()->supports_inline_contig_alloc()) { diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp --- a/hotspot/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1641,12 +1641,14 @@ } void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) { + Register len = op->len()->as_register(); + LP64_ONLY( __ movslq(len, len); ) + if (UseSlowPath || (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) || (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) { __ jmp(*op->stub()->entry()); } else { - Register len = op->len()->as_register(); Register tmp1 = op->tmp1()->as_register(); Register tmp2 = op->tmp2()->as_register(); Register tmp3 = op->tmp3()->as_register(); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/x86/vm/globals_x86.hpp --- a/hotspot/src/cpu/x86/vm/globals_x86.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/x86/vm/globals_x86.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -62,7 +62,7 @@ // due to lack of optimization caused by C++ compiler bugs define_pd_global(intx, StackShadowPages, SOLARIS_ONLY(20) NOT_SOLARIS(6) DEBUG_ONLY(+2)); #else -define_pd_global(intx, StackShadowPages, 3 DEBUG_ONLY(+1)); +define_pd_global(intx, StackShadowPages, 3 DEBUG_ONLY(+5)); #endif // AMD64 define_pd_global(intx, PreInflateSpin, 10); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/x86/vm/interp_masm_x86_32.cpp --- a/hotspot/src/cpu/x86/vm/interp_masm_x86_32.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/x86/vm/interp_masm_x86_32.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -819,7 +819,7 @@ // Set the method data pointer for the current bcp. void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() { assert(ProfileInterpreter, "must be profiling interpreter"); - Label zero_continue; + Label set_mdp; push(rax); push(rbx); @@ -827,21 +827,17 @@ // Test MDO to avoid the call if it is NULL. movptr(rax, Address(rbx, in_bytes(methodOopDesc::method_data_offset()))); testptr(rax, rax); - jcc(Assembler::zero, zero_continue); - + jcc(Assembler::zero, set_mdp); // rbx,: method // rsi: bcp call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), rbx, rsi); // rax,: mdi - + // mdo is guaranteed to be non-zero here, we checked for it before the call. movptr(rbx, Address(rbx, in_bytes(methodOopDesc::method_data_offset()))); - testptr(rbx, rbx); - jcc(Assembler::zero, zero_continue); addptr(rbx, in_bytes(methodDataOopDesc::data_offset())); - addptr(rbx, rax); - movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rbx); - - bind(zero_continue); + addptr(rax, rbx); + bind(set_mdp); + movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rax); pop(rbx); pop(rax); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/x86/vm/interp_masm_x86_64.cpp --- a/hotspot/src/cpu/x86/vm/interp_masm_x86_64.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/x86/vm/interp_masm_x86_64.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -855,7 +855,7 @@ // Set the method data pointer for the current bcp. void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() { assert(ProfileInterpreter, "must be profiling interpreter"); - Label zero_continue; + Label set_mdp; push(rax); push(rbx); @@ -863,21 +863,17 @@ // Test MDO to avoid the call if it is NULL. movptr(rax, Address(rbx, in_bytes(methodOopDesc::method_data_offset()))); testptr(rax, rax); - jcc(Assembler::zero, zero_continue); - + jcc(Assembler::zero, set_mdp); // rbx: method // r13: bcp call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), rbx, r13); // rax: mdi - + // mdo is guaranteed to be non-zero here, we checked for it before the call. movptr(rbx, Address(rbx, in_bytes(methodOopDesc::method_data_offset()))); - testptr(rbx, rbx); - jcc(Assembler::zero, zero_continue); addptr(rbx, in_bytes(methodDataOopDesc::data_offset())); - addptr(rbx, rax); - movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rbx); - - bind(zero_continue); + addptr(rax, rbx); + bind(set_mdp); + movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rax); pop(rbx); pop(rax); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/x86/vm/methodHandles_x86.cpp --- a/hotspot/src/cpu/x86/vm/methodHandles_x86.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/x86/vm/methodHandles_x86.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -390,7 +390,7 @@ // // Generate an "entry" field for a method handle. // This determines how the method handle will respond to calls. -void MethodHandles::generate_method_handle_stub(MacroAssembler* _masm, MethodHandles::EntryKind ek, TRAPS) { +void MethodHandles::generate_method_handle_stub(MacroAssembler* _masm, MethodHandles::EntryKind ek) { // Here is the register state during an interpreted call, // as set up by generate_method_handle_interpreter_entry(): // - rbx: garbage temp (was MethodHandle.invoke methodOop, unused) @@ -451,8 +451,9 @@ // exception. Since we use a C2I adapter to set up the // interpreter state, arguments are expected in compiler // argument registers. - methodHandle mh(raise_exception_method()); - address c2i_entry = methodOopDesc::make_adapters(mh, CHECK); + assert(raise_exception_method(), "must be set"); + address c2i_entry = raise_exception_method()->get_c2i_entry(); + assert(c2i_entry, "method must be linked"); const Register rdi_pc = rax; __ pop(rdi_pc); // caller PC diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/x86/vm/templateInterpreter_x86_32.cpp --- a/hotspot/src/cpu/x86/vm/templateInterpreter_x86_32.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/x86/vm/templateInterpreter_x86_32.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1367,15 +1367,9 @@ if (ProfileInterpreter) { // We have decided to profile this method in the interpreter __ bind(profile_method); - - __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method), rsi, true); - - __ movptr(rbx, Address(rbp, method_offset)); // restore methodOop - __ movptr(rax, Address(rbx, in_bytes(methodOopDesc::method_data_offset()))); - __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rax); - __ test_method_data_pointer(rax, profile_method_continue); - __ addptr(rax, in_bytes(methodDataOopDesc::data_offset())); - __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rax); + __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method)); + __ set_method_data_pointer_for_bcp(); + __ get_method(rbx); __ jmp(profile_method_continue); } // Handle overflow of counter and compile method diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/x86/vm/templateInterpreter_x86_64.cpp --- a/hotspot/src/cpu/x86/vm/templateInterpreter_x86_64.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/x86/vm/templateInterpreter_x86_64.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1383,20 +1383,9 @@ if (ProfileInterpreter) { // We have decided to profile this method in the interpreter __ bind(profile_method); - - __ call_VM(noreg, - CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method), - r13, true); - - __ movptr(rbx, Address(rbp, method_offset)); // restore methodOop - __ movptr(rax, Address(rbx, - in_bytes(methodOopDesc::method_data_offset()))); - __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), - rax); - __ test_method_data_pointer(rax, profile_method_continue); - __ addptr(rax, in_bytes(methodDataOopDesc::data_offset())); - __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), - rax); + __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method)); + __ set_method_data_pointer_for_bcp(); + __ get_method(rbx); __ jmp(profile_method_continue); } // Handle overflow of counter and compile method diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/x86/vm/templateTable_x86_32.cpp --- a/hotspot/src/cpu/x86/vm/templateTable_x86_32.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/x86/vm/templateTable_x86_32.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1665,16 +1665,9 @@ if (ProfileInterpreter) { // Out-of-line code to allocate method data oop. __ bind(profile_method); - __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method), rsi); + __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method)); __ load_unsigned_byte(rbx, Address(rsi, 0)); // restore target bytecode - __ movptr(rcx, Address(rbp, method_offset)); - __ movptr(rcx, Address(rcx, in_bytes(methodOopDesc::method_data_offset()))); - __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rcx); - __ test_method_data_pointer(rcx, dispatch); - // offset non-null mdp by MDO::data_offset() + IR::profile_method() - __ addptr(rcx, in_bytes(methodDataOopDesc::data_offset())); - __ addptr(rcx, rax); - __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rcx); + __ set_method_data_pointer_for_bcp(); __ jmp(dispatch); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/cpu/x86/vm/templateTable_x86_64.cpp --- a/hotspot/src/cpu/x86/vm/templateTable_x86_64.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/cpu/x86/vm/templateTable_x86_64.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1695,21 +1695,9 @@ if (ProfileInterpreter) { // Out-of-line code to allocate method data oop. __ bind(profile_method); - __ call_VM(noreg, - CAST_FROM_FN_PTR(address, - InterpreterRuntime::profile_method), r13); + __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method)); __ load_unsigned_byte(rbx, Address(r13, 0)); // restore target bytecode - __ movptr(rcx, Address(rbp, method_offset)); - __ movptr(rcx, Address(rcx, - in_bytes(methodOopDesc::method_data_offset()))); - __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), - rcx); - __ test_method_data_pointer(rcx, dispatch); - // offset non-null mdp by MDO::data_offset() + IR::profile_method() - __ addptr(rcx, in_bytes(methodDataOopDesc::data_offset())); - __ addptr(rcx, rax); - __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), - rcx); + __ set_method_data_pointer_for_bcp(); __ jmp(dispatch); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/os/linux/vm/os_linux.cpp --- a/hotspot/src/os/linux/vm/os_linux.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/os/linux/vm/os_linux.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1610,10 +1610,9 @@ const char* os::dll_file_extension() { return ".so"; } -const char* os::get_temp_directory() { - const char *prop = Arguments::get_property("java.io.tmpdir"); - return prop == NULL ? "/tmp" : prop; -} +// This must be hard coded because it's the system's temporary +// directory not the java application's temp directory, ala java.io.tmpdir. +const char* os::get_temp_directory() { return "/tmp"; } static bool file_exists(const char* filename) { struct stat statbuf; diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/os/solaris/vm/os_solaris.cpp --- a/hotspot/src/os/solaris/vm/os_solaris.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/os/solaris/vm/os_solaris.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1884,10 +1884,9 @@ const char* os::dll_file_extension() { return ".so"; } -const char* os::get_temp_directory() { - const char *prop = Arguments::get_property("java.io.tmpdir"); - return prop == NULL ? "/tmp" : prop; -} +// This must be hard coded because it's the system's temporary +// directory not the java application's temp directory, ala java.io.tmpdir. +const char* os::get_temp_directory() { return "/tmp"; } static bool file_exists(const char* filename) { struct stat statbuf; diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/os/windows/vm/os_windows.cpp --- a/hotspot/src/os/windows/vm/os_windows.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/os/windows/vm/os_windows.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1044,9 +1044,9 @@ return 0; } +// This must be hard coded because it's the system's temporary +// directory not the java application's temp directory, ala java.io.tmpdir. const char* os::get_temp_directory() { - const char *prop = Arguments::get_property("java.io.tmpdir"); - if (prop != 0) return prop; static char path_buf[MAX_PATH]; if (GetTempPath(MAX_PATH, path_buf)>0) return path_buf; diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/os_cpu/linux_sparc/vm/atomic_linux_sparc.inline.hpp --- a/hotspot/src/os_cpu/linux_sparc/vm/atomic_linux_sparc.inline.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/os_cpu/linux_sparc/vm/atomic_linux_sparc.inline.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -54,6 +54,8 @@ inline void Atomic::dec_ptr(volatile intptr_t* dest) { (void)add_ptr(-1, dest); } inline void Atomic::dec_ptr(volatile void* dest) { (void)add_ptr(-1, dest); } +inline jlong Atomic::load(volatile jlong* src) { return *src; } + inline jint Atomic::add (jint add_value, volatile jint* dest) { intptr_t rv; __asm__ volatile( diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/os_cpu/linux_x86/vm/atomic_linux_x86.inline.hpp --- a/hotspot/src/os_cpu/linux_x86/vm/atomic_linux_x86.inline.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/os_cpu/linux_x86/vm/atomic_linux_x86.inline.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -100,11 +100,6 @@ return exchange_value; } -extern "C" { - // defined in linux_x86.s - jlong _Atomic_cmpxchg_long(jlong, volatile jlong*, jlong, bool); -} - #ifdef AMD64 inline void Atomic::store (jlong store_value, jlong* dest) { *dest = store_value; } inline void Atomic::store (jlong store_value, volatile jlong* dest) { *dest = store_value; } @@ -164,9 +159,9 @@ return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value); } -#else -//inline void Atomic::store (jlong store_value, jlong* dest) { *dest = store_value; } -//inline void Atomic::store (jlong store_value, volatile jlong* dest) { *dest = store_value; } +inline jlong Atomic::load(volatile jlong* src) { return *src; } + +#else // !AMD64 inline intptr_t Atomic::add_ptr(intptr_t add_value, volatile intptr_t* dest) { return (intptr_t)Atomic::add((jint)add_value, (volatile jint*)dest); @@ -189,6 +184,12 @@ return (intptr_t)xchg((jint)exchange_value, (volatile jint*)dest); } +extern "C" { + // defined in linux_x86.s + jlong _Atomic_cmpxchg_long(jlong, volatile jlong*, jlong, bool); + void _Atomic_move_long(volatile jlong* src, volatile jlong* dst); +} + inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) { return _Atomic_cmpxchg_long(exchange_value, dest, compare_value, os::is_MP()); } @@ -200,6 +201,21 @@ inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) { return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value); } + +inline jlong Atomic::load(volatile jlong* src) { + volatile jlong dest; + _Atomic_move_long(src, &dest); + return dest; +} + +inline void Atomic::store(jlong store_value, jlong* dest) { + _Atomic_move_long((volatile jlong*)&store_value, (volatile jlong*)dest); +} + +inline void Atomic::store(jlong store_value, volatile jlong* dest) { + _Atomic_move_long((volatile jlong*)&store_value, dest); +} + #endif // AMD64 #endif // OS_CPU_LINUX_X86_VM_ATOMIC_LINUX_X86_INLINE_HPP diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/os_cpu/linux_x86/vm/linux_x86_32.s --- a/hotspot/src/os_cpu/linux_x86/vm/linux_x86_32.s Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/os_cpu/linux_x86/vm/linux_x86_32.s Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ # -# Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -38,6 +38,7 @@ .globl _mmx_Copy_arrayof_conjoint_jshorts .globl _Atomic_cmpxchg_long + .globl _Atomic_move_long .text @@ -653,3 +654,15 @@ popl %ebx ret + + # Support for jlong Atomic::load and Atomic::store. + # void _Atomic_move_long(volatile jlong* src, volatile jlong* dst) + .p2align 4,,15 + .type _Atomic_move_long,@function +_Atomic_move_long: + movl 4(%esp), %eax # src + fildll (%eax) + movl 8(%esp), %eax # dest + fistpll (%eax) + ret + diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/os_cpu/linux_x86/vm/orderAccess_linux_x86.inline.hpp --- a/hotspot/src/os_cpu/linux_x86/vm/orderAccess_linux_x86.inline.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/os_cpu/linux_x86/vm/orderAccess_linux_x86.inline.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,6 +25,7 @@ #ifndef OS_CPU_LINUX_X86_VM_ORDERACCESS_LINUX_X86_INLINE_HPP #define OS_CPU_LINUX_X86_VM_ORDERACCESS_LINUX_X86_INLINE_HPP +#include "runtime/atomic.hpp" #include "runtime/orderAccess.hpp" #include "vm_version_x86.hpp" @@ -64,11 +65,11 @@ inline jbyte OrderAccess::load_acquire(volatile jbyte* p) { return *p; } inline jshort OrderAccess::load_acquire(volatile jshort* p) { return *p; } inline jint OrderAccess::load_acquire(volatile jint* p) { return *p; } -inline jlong OrderAccess::load_acquire(volatile jlong* p) { return *p; } +inline jlong OrderAccess::load_acquire(volatile jlong* p) { return Atomic::load(p); } inline jubyte OrderAccess::load_acquire(volatile jubyte* p) { return *p; } inline jushort OrderAccess::load_acquire(volatile jushort* p) { return *p; } inline juint OrderAccess::load_acquire(volatile juint* p) { return *p; } -inline julong OrderAccess::load_acquire(volatile julong* p) { return *p; } +inline julong OrderAccess::load_acquire(volatile julong* p) { return Atomic::load((volatile jlong*)p); } inline jfloat OrderAccess::load_acquire(volatile jfloat* p) { return *p; } inline jdouble OrderAccess::load_acquire(volatile jdouble* p) { return *p; } @@ -79,11 +80,11 @@ inline void OrderAccess::release_store(volatile jbyte* p, jbyte v) { *p = v; } inline void OrderAccess::release_store(volatile jshort* p, jshort v) { *p = v; } inline void OrderAccess::release_store(volatile jint* p, jint v) { *p = v; } -inline void OrderAccess::release_store(volatile jlong* p, jlong v) { *p = v; } +inline void OrderAccess::release_store(volatile jlong* p, jlong v) { Atomic::store(v, p); } inline void OrderAccess::release_store(volatile jubyte* p, jubyte v) { *p = v; } inline void OrderAccess::release_store(volatile jushort* p, jushort v) { *p = v; } inline void OrderAccess::release_store(volatile juint* p, juint v) { *p = v; } -inline void OrderAccess::release_store(volatile julong* p, julong v) { *p = v; } +inline void OrderAccess::release_store(volatile julong* p, julong v) { Atomic::store((jlong)v, (volatile jlong*)p); } inline void OrderAccess::release_store(volatile jfloat* p, jfloat v) { *p = v; } inline void OrderAccess::release_store(volatile jdouble* p, jdouble v) { *p = v; } @@ -178,7 +179,7 @@ : "0" (v), "r" (p) : "memory"); #else - *p = v; fence(); + release_store(p, v); fence(); #endif // AMD64 } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/os_cpu/solaris_sparc/vm/atomic_solaris_sparc.inline.hpp --- a/hotspot/src/os_cpu/solaris_sparc/vm/atomic_solaris_sparc.inline.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/os_cpu/solaris_sparc/vm/atomic_solaris_sparc.inline.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -35,14 +35,12 @@ inline void Atomic::store (jbyte store_value, jbyte* dest) { *dest = store_value; } inline void Atomic::store (jshort store_value, jshort* dest) { *dest = store_value; } inline void Atomic::store (jint store_value, jint* dest) { *dest = store_value; } -inline void Atomic::store (jlong store_value, jlong* dest) { *dest = store_value; } inline void Atomic::store_ptr(intptr_t store_value, intptr_t* dest) { *dest = store_value; } inline void Atomic::store_ptr(void* store_value, void* dest) { *(void**)dest = store_value; } inline void Atomic::store (jbyte store_value, volatile jbyte* dest) { *dest = store_value; } inline void Atomic::store (jshort store_value, volatile jshort* dest) { *dest = store_value; } inline void Atomic::store (jint store_value, volatile jint* dest) { *dest = store_value; } -inline void Atomic::store (jlong store_value, volatile jlong* dest) { *dest = store_value; } inline void Atomic::store_ptr(intptr_t store_value, volatile intptr_t* dest) { *dest = store_value; } inline void Atomic::store_ptr(void* store_value, volatile void* dest) { *(void* volatile *)dest = store_value; } @@ -54,8 +52,49 @@ inline void Atomic::dec_ptr(volatile intptr_t* dest) { (void)add_ptr(-1, dest); } inline void Atomic::dec_ptr(volatile void* dest) { (void)add_ptr(-1, dest); } + +#ifdef _LP64 + +inline void Atomic::store(jlong store_value, jlong* dest) { *dest = store_value; } +inline void Atomic::store(jlong store_value, volatile jlong* dest) { *dest = store_value; } inline jlong Atomic::load(volatile jlong* src) { return *src; } +#else + +extern "C" void _Atomic_move_long_v8(volatile jlong* src, volatile jlong* dst); +extern "C" void _Atomic_move_long_v9(volatile jlong* src, volatile jlong* dst); + +inline void Atomic_move_long(volatile jlong* src, volatile jlong* dst) { +#ifdef COMPILER2 + // Compiler2 does not support v8, it is used only for v9. + assert (VM_Version::v9_instructions_work(), "only supported on v9"); + _Atomic_move_long_v9(src, dst); +#else + // The branch is cheaper then emulated LDD. + if (VM_Version::v9_instructions_work()) { + _Atomic_move_long_v9(src, dst); + } else { + _Atomic_move_long_v8(src, dst); + } +#endif +} + +inline jlong Atomic::load(volatile jlong* src) { + volatile jlong dest; + Atomic_move_long(src, &dest); + return dest; +} + +inline void Atomic::store(jlong store_value, jlong* dest) { + Atomic_move_long((volatile jlong*)&store_value, (volatile jlong*)dest); +} + +inline void Atomic::store(jlong store_value, volatile jlong* dest) { + Atomic_move_long((volatile jlong*)&store_value, dest); +} + +#endif + #ifdef _GNU_SOURCE inline jint Atomic::add (jint add_value, volatile jint* dest) { diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/os_cpu/solaris_sparc/vm/orderAccess_solaris_sparc.inline.hpp --- a/hotspot/src/os_cpu/solaris_sparc/vm/orderAccess_solaris_sparc.inline.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/os_cpu/solaris_sparc/vm/orderAccess_solaris_sparc.inline.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -77,11 +77,11 @@ inline jbyte OrderAccess::load_acquire(volatile jbyte* p) { return *p; } inline jshort OrderAccess::load_acquire(volatile jshort* p) { return *p; } inline jint OrderAccess::load_acquire(volatile jint* p) { return *p; } -inline jlong OrderAccess::load_acquire(volatile jlong* p) { return *p; } +inline jlong OrderAccess::load_acquire(volatile jlong* p) { return Atomic::load(p); } inline jubyte OrderAccess::load_acquire(volatile jubyte* p) { return *p; } inline jushort OrderAccess::load_acquire(volatile jushort* p) { return *p; } inline juint OrderAccess::load_acquire(volatile juint* p) { return *p; } -inline julong OrderAccess::load_acquire(volatile julong* p) { return *p; } +inline julong OrderAccess::load_acquire(volatile julong* p) { return Atomic::load((volatile jlong*)p); } inline jfloat OrderAccess::load_acquire(volatile jfloat* p) { return *p; } inline jdouble OrderAccess::load_acquire(volatile jdouble* p) { return *p; } @@ -92,11 +92,11 @@ inline void OrderAccess::release_store(volatile jbyte* p, jbyte v) { *p = v; } inline void OrderAccess::release_store(volatile jshort* p, jshort v) { *p = v; } inline void OrderAccess::release_store(volatile jint* p, jint v) { *p = v; } -inline void OrderAccess::release_store(volatile jlong* p, jlong v) { *p = v; } +inline void OrderAccess::release_store(volatile jlong* p, jlong v) { Atomic::store(v, p); } inline void OrderAccess::release_store(volatile jubyte* p, jubyte v) { *p = v; } inline void OrderAccess::release_store(volatile jushort* p, jushort v) { *p = v; } inline void OrderAccess::release_store(volatile juint* p, juint v) { *p = v; } -inline void OrderAccess::release_store(volatile julong* p, julong v) { *p = v; } +inline void OrderAccess::release_store(volatile julong* p, julong v) { Atomic::store((jlong)v, (volatile jlong*)p); } inline void OrderAccess::release_store(volatile jfloat* p, jfloat v) { *p = v; } inline void OrderAccess::release_store(volatile jdouble* p, jdouble v) { *p = v; } @@ -120,11 +120,11 @@ inline void OrderAccess::release_store_fence(volatile jbyte* p, jbyte v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile jshort* p, jshort v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile jint* p, jint v) { *p = v; fence(); } -inline void OrderAccess::release_store_fence(volatile jlong* p, jlong v) { *p = v; fence(); } +inline void OrderAccess::release_store_fence(volatile jlong* p, jlong v) { release_store(p, v); fence(); } inline void OrderAccess::release_store_fence(volatile jubyte* p, jubyte v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile jushort* p, jushort v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile juint* p, juint v) { *p = v; fence(); } -inline void OrderAccess::release_store_fence(volatile julong* p, julong v) { *p = v; fence(); } +inline void OrderAccess::release_store_fence(volatile julong* p, julong v) { release_store(p, v); fence(); } inline void OrderAccess::release_store_fence(volatile jfloat* p, jfloat v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile jdouble* p, jdouble v) { *p = v; fence(); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/os_cpu/solaris_sparc/vm/solaris_sparc.il --- a/hotspot/src/os_cpu/solaris_sparc/vm/solaris_sparc.il Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/os_cpu/solaris_sparc/vm/solaris_sparc.il Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ // -// Copyright (c) 2002, 2005, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2002, 2011, Oracle and/or its affiliates. All rights reserved. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -152,6 +152,39 @@ .nonvolatile .end + // Support for jlong Atomic::load and Atomic::store on v8. + // + // void _Atomic_move_long_v8(volatile jlong* src, volatile jlong* dst) + // + // Arguments: + // src: O0 + // dest: O1 + // + // Overwrites O2 and O3 + + .inline _Atomic_move_long_v8,2 + .volatile + ldd [%o0], %o2 + std %o2, [%o1] + .nonvolatile + .end + + // Support for jlong Atomic::load and Atomic::store on v9. + // + // void _Atomic_move_long_v9(volatile jlong* src, volatile jlong* dst) + // + // Arguments: + // src: O0 + // dest: O1 + // + // Overwrites O2 + + .inline _Atomic_move_long_v9,2 + .volatile + ldx [%o0], %o2 + stx %o2, [%o1] + .nonvolatile + .end // Support for jint Atomic::add(jint add_value, volatile jint* dest). // diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/os_cpu/solaris_x86/vm/atomic_solaris_x86.inline.hpp --- a/hotspot/src/os_cpu/solaris_x86/vm/atomic_solaris_x86.inline.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/os_cpu/solaris_x86/vm/atomic_solaris_x86.inline.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -151,14 +151,22 @@ return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value); } -extern "C" void _Atomic_load_long(volatile jlong* src, volatile jlong* dst); +extern "C" void _Atomic_move_long(volatile jlong* src, volatile jlong* dst); inline jlong Atomic::load(volatile jlong* src) { volatile jlong dest; - _Atomic_load_long(src, &dest); + _Atomic_move_long(src, &dest); return dest; } +inline void Atomic::store(jlong store_value, jlong* dest) { + _Atomic_move_long((volatile jlong*)&store_value, (volatile jlong*)dest); +} + +inline void Atomic::store(jlong store_value, volatile jlong* dest) { + _Atomic_move_long((volatile jlong*)&store_value, dest); +} + #endif // AMD64 #ifdef _GNU_SOURCE diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/os_cpu/solaris_x86/vm/orderAccess_solaris_x86.inline.hpp --- a/hotspot/src/os_cpu/solaris_x86/vm/orderAccess_solaris_x86.inline.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/os_cpu/solaris_x86/vm/orderAccess_solaris_x86.inline.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,6 +25,7 @@ #ifndef OS_CPU_SOLARIS_X86_VM_ORDERACCESS_SOLARIS_X86_INLINE_HPP #define OS_CPU_SOLARIS_X86_VM_ORDERACCESS_SOLARIS_X86_INLINE_HPP +#include "runtime/atomic.hpp" #include "runtime/orderAccess.hpp" #include "vm_version_x86.hpp" @@ -80,11 +81,11 @@ inline jbyte OrderAccess::load_acquire(volatile jbyte* p) { return *p; } inline jshort OrderAccess::load_acquire(volatile jshort* p) { return *p; } inline jint OrderAccess::load_acquire(volatile jint* p) { return *p; } -inline jlong OrderAccess::load_acquire(volatile jlong* p) { return *p; } +inline jlong OrderAccess::load_acquire(volatile jlong* p) { return Atomic::load(p); } inline jubyte OrderAccess::load_acquire(volatile jubyte* p) { return *p; } inline jushort OrderAccess::load_acquire(volatile jushort* p) { return *p; } inline juint OrderAccess::load_acquire(volatile juint* p) { return *p; } -inline julong OrderAccess::load_acquire(volatile julong* p) { return *p; } +inline julong OrderAccess::load_acquire(volatile julong* p) { return Atomic::load((volatile jlong*)p); } inline jfloat OrderAccess::load_acquire(volatile jfloat* p) { return *p; } inline jdouble OrderAccess::load_acquire(volatile jdouble* p) { return *p; } @@ -95,11 +96,11 @@ inline void OrderAccess::release_store(volatile jbyte* p, jbyte v) { *p = v; } inline void OrderAccess::release_store(volatile jshort* p, jshort v) { *p = v; } inline void OrderAccess::release_store(volatile jint* p, jint v) { *p = v; } -inline void OrderAccess::release_store(volatile jlong* p, jlong v) { *p = v; } +inline void OrderAccess::release_store(volatile jlong* p, jlong v) { Atomic::store(v, p); } inline void OrderAccess::release_store(volatile jubyte* p, jubyte v) { *p = v; } inline void OrderAccess::release_store(volatile jushort* p, jushort v) { *p = v; } inline void OrderAccess::release_store(volatile juint* p, juint v) { *p = v; } -inline void OrderAccess::release_store(volatile julong* p, julong v) { *p = v; } +inline void OrderAccess::release_store(volatile julong* p, julong v) { Atomic::store((jlong)v, (volatile jlong*)p); } inline void OrderAccess::release_store(volatile jfloat* p, jfloat v) { *p = v; } inline void OrderAccess::release_store(volatile jdouble* p, jdouble v) { *p = v; } @@ -123,11 +124,11 @@ inline void OrderAccess::release_store_fence(volatile jbyte* p, jbyte v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile jshort* p, jshort v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile jint* p, jint v) { *p = v; fence(); } -inline void OrderAccess::release_store_fence(volatile jlong* p, jlong v) { *p = v; fence(); } +inline void OrderAccess::release_store_fence(volatile jlong* p, jlong v) { release_store(p, v); fence(); } inline void OrderAccess::release_store_fence(volatile jubyte* p, jubyte v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile jushort* p, jushort v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile juint* p, juint v) { *p = v; fence(); } -inline void OrderAccess::release_store_fence(volatile julong* p, julong v) { *p = v; fence(); } +inline void OrderAccess::release_store_fence(volatile julong* p, julong v) { release_store(p, v); fence(); } inline void OrderAccess::release_store_fence(volatile jfloat* p, jfloat v) { *p = v; fence(); } inline void OrderAccess::release_store_fence(volatile jdouble* p, jdouble v) { *p = v; fence(); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/os_cpu/solaris_x86/vm/solaris_x86_32.il --- a/hotspot/src/os_cpu/solaris_x86/vm/solaris_x86_32.il Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/os_cpu/solaris_x86/vm/solaris_x86_32.il Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ // -// Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -104,8 +104,9 @@ popl %ebx .end - // Support for void Atomic::load(volatile jlong* src, volatile jlong* dest). - .inline _Atomic_load_long,2 + // Support for jlong Atomic::load and Atomic::store. + // void _Atomic_move_long(volatile jlong* src, volatile jlong* dst) + .inline _Atomic_move_long,2 movl 0(%esp), %eax // src fildll (%eax) movl 4(%esp), %eax // dest diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/os_cpu/windows_x86/vm/atomic_windows_x86.inline.hpp --- a/hotspot/src/os_cpu/windows_x86/vm/atomic_windows_x86.inline.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/os_cpu/windows_x86/vm/atomic_windows_x86.inline.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -137,10 +137,10 @@ return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value); } +inline jlong Atomic::load(volatile jlong* src) { return *src; } + #else // !AMD64 -//inline void Atomic::store (jlong store_value, jlong* dest) { *dest = store_value; } -//inline void Atomic::store (jlong store_value, volatile jlong* dest) { *dest = store_value; } inline jint Atomic::add (jint add_value, volatile jint* dest) { int mp = os::is_MP(); __asm { @@ -254,6 +254,33 @@ inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) { return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value); } + +inline jlong Atomic::load(volatile jlong* src) { + volatile jlong dest; + volatile jlong* pdest = &dest; + __asm { + mov eax, src + fild qword ptr [eax] + mov eax, pdest + fistp qword ptr [eax] + } + return dest; +} + +inline void Atomic::store(jlong store_value, volatile jlong* dest) { + volatile jlong* src = &store_value; + __asm { + mov eax, src + fild qword ptr [eax] + mov eax, dest + fistp qword ptr [eax] + } +} + +inline void Atomic::store(jlong store_value, jlong* dest) { + Atomic::store(store_value, (volatile jlong*)dest); +} + #endif // AMD64 #pragma warning(default: 4035) // Enables warnings reporting missing return statement diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/os_cpu/windows_x86/vm/orderAccess_windows_x86.inline.hpp --- a/hotspot/src/os_cpu/windows_x86/vm/orderAccess_windows_x86.inline.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/os_cpu/windows_x86/vm/orderAccess_windows_x86.inline.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,6 +25,7 @@ #ifndef OS_CPU_WINDOWS_X86_VM_ORDERACCESS_WINDOWS_X86_INLINE_HPP #define OS_CPU_WINDOWS_X86_VM_ORDERACCESS_WINDOWS_X86_INLINE_HPP +#include "runtime/atomic.hpp" #include "runtime/orderAccess.hpp" #include "vm_version_x86.hpp" @@ -65,11 +66,11 @@ inline jbyte OrderAccess::load_acquire(volatile jbyte* p) { return *p; } inline jshort OrderAccess::load_acquire(volatile jshort* p) { return *p; } inline jint OrderAccess::load_acquire(volatile jint* p) { return *p; } -inline jlong OrderAccess::load_acquire(volatile jlong* p) { return *p; } +inline jlong OrderAccess::load_acquire(volatile jlong* p) { return Atomic::load(p); } inline jubyte OrderAccess::load_acquire(volatile jubyte* p) { return *p; } inline jushort OrderAccess::load_acquire(volatile jushort* p) { return *p; } inline juint OrderAccess::load_acquire(volatile juint* p) { return *p; } -inline julong OrderAccess::load_acquire(volatile julong* p) { return *p; } +inline julong OrderAccess::load_acquire(volatile julong* p) { return Atomic::load((volatile jlong*)p); } inline jfloat OrderAccess::load_acquire(volatile jfloat* p) { return *p; } inline jdouble OrderAccess::load_acquire(volatile jdouble* p) { return *p; } @@ -80,11 +81,11 @@ inline void OrderAccess::release_store(volatile jbyte* p, jbyte v) { *p = v; } inline void OrderAccess::release_store(volatile jshort* p, jshort v) { *p = v; } inline void OrderAccess::release_store(volatile jint* p, jint v) { *p = v; } -inline void OrderAccess::release_store(volatile jlong* p, jlong v) { *p = v; } +inline void OrderAccess::release_store(volatile jlong* p, jlong v) { Atomic::store(v, p); } inline void OrderAccess::release_store(volatile jubyte* p, jubyte v) { *p = v; } inline void OrderAccess::release_store(volatile jushort* p, jushort v) { *p = v; } inline void OrderAccess::release_store(volatile juint* p, juint v) { *p = v; } -inline void OrderAccess::release_store(volatile julong* p, julong v) { *p = v; } +inline void OrderAccess::release_store(volatile julong* p, julong v) { Atomic::store((jlong)v, (volatile jlong*)p); } inline void OrderAccess::release_store(volatile jfloat* p, jfloat v) { *p = v; } inline void OrderAccess::release_store(volatile jdouble* p, jdouble v) { *p = v; } @@ -188,7 +189,7 @@ #endif // AMD64 } -inline void OrderAccess::release_store_fence(volatile jlong* p, jlong v) { *p = v; fence(); } +inline void OrderAccess::release_store_fence(volatile jlong* p, jlong v) { release_store(p, v); fence(); } inline void OrderAccess::release_store_fence(volatile jubyte* p, jubyte v) { release_store_fence((volatile jbyte*)p, (jbyte)v); } inline void OrderAccess::release_store_fence(volatile jushort* p, jushort v) { release_store_fence((volatile jshort*)p, (jshort)v); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/c1/c1_GraphBuilder.cpp --- a/hotspot/src/share/vm/c1/c1_GraphBuilder.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/c1/c1_GraphBuilder.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -319,24 +319,24 @@ case Bytecodes::_tableswitch: { // set block for each case - Bytecode_tableswitch *switch_ = Bytecode_tableswitch_at(s.cur_bcp()); - int l = switch_->length(); + Bytecode_tableswitch sw(&s); + int l = sw.length(); for (int i = 0; i < l; i++) { - make_block_at(cur_bci + switch_->dest_offset_at(i), current); + make_block_at(cur_bci + sw.dest_offset_at(i), current); } - make_block_at(cur_bci + switch_->default_offset(), current); + make_block_at(cur_bci + sw.default_offset(), current); current = NULL; break; } case Bytecodes::_lookupswitch: { // set block for each case - Bytecode_lookupswitch *switch_ = Bytecode_lookupswitch_at(s.cur_bcp()); - int l = switch_->number_of_pairs(); + Bytecode_lookupswitch sw(&s); + int l = sw.number_of_pairs(); for (int i = 0; i < l; i++) { - make_block_at(cur_bci + switch_->pair_at(i)->offset(), current); + make_block_at(cur_bci + sw.pair_at(i).offset(), current); } - make_block_at(cur_bci + switch_->default_offset(), current); + make_block_at(cur_bci + sw.default_offset(), current); current = NULL; break; } @@ -1275,15 +1275,15 @@ void GraphBuilder::table_switch() { - Bytecode_tableswitch* switch_ = Bytecode_tableswitch_at(method()->code() + bci()); - const int l = switch_->length(); + Bytecode_tableswitch sw(stream()); + const int l = sw.length(); if (CanonicalizeNodes && l == 1) { // total of 2 successors => use If instead of switch // Note: This code should go into the canonicalizer as soon as it can // can handle canonicalized forms that contain more than one node. - Value key = append(new Constant(new IntConstant(switch_->low_key()))); - BlockBegin* tsux = block_at(bci() + switch_->dest_offset_at(0)); - BlockBegin* fsux = block_at(bci() + switch_->default_offset()); + Value key = append(new Constant(new IntConstant(sw.low_key()))); + BlockBegin* tsux = block_at(bci() + sw.dest_offset_at(0)); + BlockBegin* fsux = block_at(bci() + sw.default_offset()); bool is_bb = tsux->bci() < bci() || fsux->bci() < bci(); ValueStack* state_before = is_bb ? copy_state_before() : NULL; append(new If(ipop(), If::eql, true, key, tsux, fsux, state_before, is_bb)); @@ -1293,29 +1293,29 @@ int i; bool has_bb = false; for (i = 0; i < l; i++) { - sux->at_put(i, block_at(bci() + switch_->dest_offset_at(i))); - if (switch_->dest_offset_at(i) < 0) has_bb = true; + sux->at_put(i, block_at(bci() + sw.dest_offset_at(i))); + if (sw.dest_offset_at(i) < 0) has_bb = true; } // add default successor - sux->at_put(i, block_at(bci() + switch_->default_offset())); + sux->at_put(i, block_at(bci() + sw.default_offset())); ValueStack* state_before = has_bb ? copy_state_before() : NULL; - append(new TableSwitch(ipop(), sux, switch_->low_key(), state_before, has_bb)); + append(new TableSwitch(ipop(), sux, sw.low_key(), state_before, has_bb)); } } void GraphBuilder::lookup_switch() { - Bytecode_lookupswitch* switch_ = Bytecode_lookupswitch_at(method()->code() + bci()); - const int l = switch_->number_of_pairs(); + Bytecode_lookupswitch sw(stream()); + const int l = sw.number_of_pairs(); if (CanonicalizeNodes && l == 1) { // total of 2 successors => use If instead of switch // Note: This code should go into the canonicalizer as soon as it can // can handle canonicalized forms that contain more than one node. // simplify to If - LookupswitchPair* pair = switch_->pair_at(0); - Value key = append(new Constant(new IntConstant(pair->match()))); - BlockBegin* tsux = block_at(bci() + pair->offset()); - BlockBegin* fsux = block_at(bci() + switch_->default_offset()); + LookupswitchPair pair = sw.pair_at(0); + Value key = append(new Constant(new IntConstant(pair.match()))); + BlockBegin* tsux = block_at(bci() + pair.offset()); + BlockBegin* fsux = block_at(bci() + sw.default_offset()); bool is_bb = tsux->bci() < bci() || fsux->bci() < bci(); ValueStack* state_before = is_bb ? copy_state_before() : NULL; append(new If(ipop(), If::eql, true, key, tsux, fsux, state_before, is_bb)); @@ -1326,13 +1326,13 @@ int i; bool has_bb = false; for (i = 0; i < l; i++) { - LookupswitchPair* pair = switch_->pair_at(i); - if (pair->offset() < 0) has_bb = true; - sux->at_put(i, block_at(bci() + pair->offset())); - keys->at_put(i, pair->match()); + LookupswitchPair pair = sw.pair_at(i); + if (pair.offset() < 0) has_bb = true; + sux->at_put(i, block_at(bci() + pair.offset())); + keys->at_put(i, pair.match()); } // add default successor - sux->at_put(i, block_at(bci() + switch_->default_offset())); + sux->at_put(i, block_at(bci() + sw.default_offset())); ValueStack* state_before = has_bb ? copy_state_before() : NULL; append(new LookupSwitch(ipop(), sux, keys, state_before, has_bb)); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/c1/c1_LIRGenerator.cpp --- a/hotspot/src/share/vm/c1/c1_LIRGenerator.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/c1/c1_LIRGenerator.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1990,9 +1990,8 @@ LIR_Opr reg = reg = rlock_result(x, x->basic_type()); + get_Object_unsafe(reg, src.result(), off.result(), type, x->is_volatile()); if (x->is_volatile() && os::is_MP()) __ membar_acquire(); - get_Object_unsafe(reg, src.result(), off.result(), type, x->is_volatile()); - if (x->is_volatile() && os::is_MP()) __ membar(); } @@ -2014,6 +2013,7 @@ if (x->is_volatile() && os::is_MP()) __ membar_release(); put_Object_unsafe(src.result(), off.result(), data.result(), type, x->is_volatile()); + if (x->is_volatile() && os::is_MP()) __ membar(); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/c1/c1_Runtime1.cpp --- a/hotspot/src/share/vm/c1/c1_Runtime1.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/c1/c1_Runtime1.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -369,7 +369,7 @@ if (branch_bci != InvocationEntryBci) { // Compute desination bci address pc = method()->code_base() + branch_bci; - Bytecodes::Code branch = Bytecodes::code_at(pc, method()); + Bytecodes::Code branch = Bytecodes::code_at(method(), pc); int offset = 0; switch (branch) { case Bytecodes::_if_icmplt: case Bytecodes::_iflt: @@ -659,14 +659,14 @@ static klassOop resolve_field_return_klass(methodHandle caller, int bci, TRAPS) { - Bytecode_field* field_access = Bytecode_field_at(caller, bci); + Bytecode_field field_access(caller, bci); // This can be static or non-static field access - Bytecodes::Code code = field_access->code(); + Bytecodes::Code code = field_access.code(); // We must load class, initialize class and resolvethe field FieldAccessInfo result; // initialize class if needed constantPoolHandle constants(THREAD, caller->constants()); - LinkResolver::resolve_field(result, constants, field_access->index(), Bytecodes::java_code(code), false, CHECK_NULL); + LinkResolver::resolve_field(result, constants, field_access.index(), Bytecodes::java_code(code), false, CHECK_NULL); return result.klass()(); } @@ -767,7 +767,7 @@ Events::log("patch_code @ " INTPTR_FORMAT , caller_frame.pc()); - Bytecodes::Code code = Bytecode_at(caller_method->bcp_from(bci))->java_code(); + Bytecodes::Code code = caller_method()->java_code_at(bci); #ifndef PRODUCT // this is used by assertions in the access_field_patching_id @@ -779,11 +779,11 @@ Handle load_klass(THREAD, NULL); // oop needed by load_klass_patching code if (stub_id == Runtime1::access_field_patching_id) { - Bytecode_field* field_access = Bytecode_field_at(caller_method, bci); + Bytecode_field field_access(caller_method, bci); FieldAccessInfo result; // initialize class if needed - Bytecodes::Code code = field_access->code(); + Bytecodes::Code code = field_access.code(); constantPoolHandle constants(THREAD, caller_method->constants()); - LinkResolver::resolve_field(result, constants, field_access->index(), Bytecodes::java_code(code), false, CHECK); + LinkResolver::resolve_field(result, constants, field_access.index(), Bytecodes::java_code(code), false, CHECK); patch_field_offset = result.field_offset(); // If we're patching a field which is volatile then at compile it @@ -811,36 +811,36 @@ } break; case Bytecodes::_new: - { Bytecode_new* bnew = Bytecode_new_at(caller_method->bcp_from(bci)); - k = caller_method->constants()->klass_at(bnew->index(), CHECK); + { Bytecode_new bnew(caller_method(), caller_method->bcp_from(bci)); + k = caller_method->constants()->klass_at(bnew.index(), CHECK); } break; case Bytecodes::_multianewarray: - { Bytecode_multianewarray* mna = Bytecode_multianewarray_at(caller_method->bcp_from(bci)); - k = caller_method->constants()->klass_at(mna->index(), CHECK); + { Bytecode_multianewarray mna(caller_method(), caller_method->bcp_from(bci)); + k = caller_method->constants()->klass_at(mna.index(), CHECK); } break; case Bytecodes::_instanceof: - { Bytecode_instanceof* io = Bytecode_instanceof_at(caller_method->bcp_from(bci)); - k = caller_method->constants()->klass_at(io->index(), CHECK); + { Bytecode_instanceof io(caller_method(), caller_method->bcp_from(bci)); + k = caller_method->constants()->klass_at(io.index(), CHECK); } break; case Bytecodes::_checkcast: - { Bytecode_checkcast* cc = Bytecode_checkcast_at(caller_method->bcp_from(bci)); - k = caller_method->constants()->klass_at(cc->index(), CHECK); + { Bytecode_checkcast cc(caller_method(), caller_method->bcp_from(bci)); + k = caller_method->constants()->klass_at(cc.index(), CHECK); } break; case Bytecodes::_anewarray: - { Bytecode_anewarray* anew = Bytecode_anewarray_at(caller_method->bcp_from(bci)); - klassOop ek = caller_method->constants()->klass_at(anew->index(), CHECK); + { Bytecode_anewarray anew(caller_method(), caller_method->bcp_from(bci)); + klassOop ek = caller_method->constants()->klass_at(anew.index(), CHECK); k = Klass::cast(ek)->array_klass(CHECK); } break; case Bytecodes::_ldc: case Bytecodes::_ldc_w: { - Bytecode_loadconstant* cc = Bytecode_loadconstant_at(caller_method, bci); - k = cc->resolve_constant(CHECK); + Bytecode_loadconstant cc(caller_method, bci); + k = cc.resolve_constant(CHECK); assert(k != NULL && !k->is_klass(), "must be class mirror or other Java constant"); } break; diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/ci/bcEscapeAnalyzer.cpp --- a/hotspot/src/share/vm/ci/bcEscapeAnalyzer.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/ci/bcEscapeAnalyzer.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -761,15 +761,15 @@ case Bytecodes::_tableswitch: { state.spop(); - Bytecode_tableswitch* switch_ = Bytecode_tableswitch_at(s.cur_bcp()); - int len = switch_->length(); + Bytecode_tableswitch sw(&s); + int len = sw.length(); int dest_bci; for (int i = 0; i < len; i++) { - dest_bci = s.cur_bci() + switch_->dest_offset_at(i); + dest_bci = s.cur_bci() + sw.dest_offset_at(i); assert(_methodBlocks->is_block_start(dest_bci), "branch destination must start a block"); successors.push(_methodBlocks->block_containing(dest_bci)); } - dest_bci = s.cur_bci() + switch_->default_offset(); + dest_bci = s.cur_bci() + sw.default_offset(); assert(_methodBlocks->is_block_start(dest_bci), "branch destination must start a block"); successors.push(_methodBlocks->block_containing(dest_bci)); assert(s.next_bci() == limit_bci, "branch must end block"); @@ -779,15 +779,15 @@ case Bytecodes::_lookupswitch: { state.spop(); - Bytecode_lookupswitch* switch_ = Bytecode_lookupswitch_at(s.cur_bcp()); - int len = switch_->number_of_pairs(); + Bytecode_lookupswitch sw(&s); + int len = sw.number_of_pairs(); int dest_bci; for (int i = 0; i < len; i++) { - dest_bci = s.cur_bci() + switch_->pair_at(i)->offset(); + dest_bci = s.cur_bci() + sw.pair_at(i).offset(); assert(_methodBlocks->is_block_start(dest_bci), "branch destination must start a block"); successors.push(_methodBlocks->block_containing(dest_bci)); } - dest_bci = s.cur_bci() + switch_->default_offset(); + dest_bci = s.cur_bci() + sw.default_offset(); assert(_methodBlocks->is_block_start(dest_bci), "branch destination must start a block"); successors.push(_methodBlocks->block_containing(dest_bci)); fall_through = false; diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/ci/ciEnv.cpp --- a/hotspot/src/share/vm/ci/ciEnv.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/ci/ciEnv.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -409,15 +409,15 @@ } else { fail_type = _unloaded_ciinstance_klass; } - klassOop found_klass; + KlassHandle found_klass; if (!require_local) { - found_klass = - SystemDictionary::find_constrained_instance_or_array_klass(sym, loader, - KILL_COMPILE_ON_FATAL_(fail_type)); + klassOop kls = SystemDictionary::find_constrained_instance_or_array_klass( + sym, loader, KILL_COMPILE_ON_FATAL_(fail_type)); + found_klass = KlassHandle(THREAD, kls); } else { - found_klass = - SystemDictionary::find_instance_or_array_klass(sym, loader, domain, - KILL_COMPILE_ON_FATAL_(fail_type)); + klassOop kls = SystemDictionary::find_instance_or_array_klass( + sym, loader, domain, KILL_COMPILE_ON_FATAL_(fail_type)); + found_klass = KlassHandle(THREAD, kls); } // If we fail to find an array klass, look again for its element type. @@ -444,9 +444,9 @@ } } - if (found_klass != NULL) { + if (found_klass() != NULL) { // Found it. Build a CI handle. - return get_object(found_klass)->as_klass(); + return get_object(found_klass())->as_klass(); } if (require_local) return NULL; diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/ci/ciMethod.hpp --- a/hotspot/src/share/vm/ci/ciMethod.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/ci/ciMethod.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -144,7 +144,7 @@ Bytecodes::Code java_code_at_bci(int bci) { address bcp = code() + bci; - return Bytecodes::java_code_at(bcp); + return Bytecodes::java_code_at(NULL, bcp); } BCEscapeAnalyzer *get_bcea(); ciMethodBlocks *get_method_blocks(); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/ci/ciMethodBlocks.cpp --- a/hotspot/src/share/vm/ci/ciMethodBlocks.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/ci/ciMethodBlocks.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2006, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -175,15 +175,15 @@ case Bytecodes::_tableswitch : { cur_block->set_control_bci(bci); - Bytecode_tableswitch* switch_ = Bytecode_tableswitch_at(s.cur_bcp()); - int len = switch_->length(); + Bytecode_tableswitch sw(&s); + int len = sw.length(); ciBlock *dest; int dest_bci; for (int i = 0; i < len; i++) { - dest_bci = s.cur_bci() + switch_->dest_offset_at(i); + dest_bci = s.cur_bci() + sw.dest_offset_at(i); dest = make_block_at(dest_bci); } - dest_bci = s.cur_bci() + switch_->default_offset(); + dest_bci = s.cur_bci() + sw.default_offset(); make_block_at(dest_bci); if (s.next_bci() < limit_bci) { dest = make_block_at(s.next_bci()); @@ -194,15 +194,15 @@ case Bytecodes::_lookupswitch: { cur_block->set_control_bci(bci); - Bytecode_lookupswitch* switch_ = Bytecode_lookupswitch_at(s.cur_bcp()); - int len = switch_->number_of_pairs(); + Bytecode_lookupswitch sw(&s); + int len = sw.number_of_pairs(); ciBlock *dest; int dest_bci; for (int i = 0; i < len; i++) { - dest_bci = s.cur_bci() + switch_->pair_at(i)->offset(); + dest_bci = s.cur_bci() + sw.pair_at(i).offset(); dest = make_block_at(dest_bci); } - dest_bci = s.cur_bci() + switch_->default_offset(); + dest_bci = s.cur_bci() + sw.default_offset(); dest = make_block_at(dest_bci); if (s.next_bci() < limit_bci) { dest = make_block_at(s.next_bci()); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/ci/ciMethodHandle.cpp --- a/hotspot/src/share/vm/ci/ciMethodHandle.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/ci/ciMethodHandle.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2009, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -38,11 +38,12 @@ // Return an adapter for this MethodHandle. ciMethod* ciMethodHandle::get_adapter(bool is_invokedynamic) const { VM_ENTRY_MARK; - Handle h(get_oop()); methodHandle callee(_callee->get_methodOop()); - MethodHandleCompiler mhc(h, callee, is_invokedynamic, THREAD); - methodHandle m = mhc.compile(CHECK_NULL); + // We catch all exceptions here that could happen in the method + // handle compiler and stop the VM. + MethodHandleCompiler mhc(h, callee, is_invokedynamic, CATCH); + methodHandle m = mhc.compile(CATCH); return CURRENT_ENV->get_object(m())->as_method(); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/ci/ciStreams.hpp --- a/hotspot/src/share/vm/ci/ciStreams.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/ci/ciStreams.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -78,8 +78,8 @@ else { assert(!is_wide(), "must not be a wide instruction"); } } - Bytecode* bytecode() const { return Bytecode_at(_bc_start); } - Bytecode* next_bytecode() const { return Bytecode_at(_pc); } + Bytecode bytecode() const { return Bytecode(this, _bc_start); } + Bytecode next_bytecode() const { return Bytecode(this, _pc); } public: // End-Of-Bytecodes @@ -151,11 +151,11 @@ bool has_cache_index() const { return Bytecodes::uses_cp_cache(cur_bc_raw()); } int get_index_u1() const { - return bytecode()->get_index_u1(cur_bc_raw()); + return bytecode().get_index_u1(cur_bc_raw()); } int get_index_u1_cpcache() const { - return bytecode()->get_index_u1_cpcache(cur_bc_raw()); + return bytecode().get_index_u1_cpcache(cur_bc_raw()); } // Get a byte index following this bytecode. @@ -169,29 +169,29 @@ // Get 2-byte index (byte swapping depending on which bytecode) int get_index_u2(bool is_wide = false) const { - return bytecode()->get_index_u2(cur_bc_raw(), is_wide); + return bytecode().get_index_u2(cur_bc_raw(), is_wide); } // Get 2-byte index in native byte order. (Rewriter::rewrite makes these.) int get_index_u2_cpcache() const { - return bytecode()->get_index_u2_cpcache(cur_bc_raw()); + return bytecode().get_index_u2_cpcache(cur_bc_raw()); } // Get 4-byte index, for invokedynamic. int get_index_u4() const { - return bytecode()->get_index_u4(cur_bc_raw()); + return bytecode().get_index_u4(cur_bc_raw()); } bool has_index_u4() const { - return bytecode()->has_index_u4(cur_bc_raw()); + return bytecode().has_index_u4(cur_bc_raw()); } // Get dimensions byte (multinewarray) int get_dimensions() const { return *(unsigned char*)(_pc-1); } // Sign-extended index byte/short, no widening - int get_constant_u1() const { return bytecode()->get_constant_u1(instruction_size()-1, cur_bc_raw()); } - int get_constant_u2(bool is_wide = false) const { return bytecode()->get_constant_u2(instruction_size()-2, cur_bc_raw(), is_wide); } + int get_constant_u1() const { return bytecode().get_constant_u1(instruction_size()-1, cur_bc_raw()); } + int get_constant_u2(bool is_wide = false) const { return bytecode().get_constant_u2(instruction_size()-2, cur_bc_raw(), is_wide); } // Get a byte signed constant for "iinc". Invalid for other bytecodes. // If prefixed with a wide bytecode, get a wide constant @@ -199,18 +199,18 @@ // 2-byte branch offset from current pc int get_dest() const { - return cur_bci() + bytecode()->get_offset_s2(cur_bc_raw()); + return cur_bci() + bytecode().get_offset_s2(cur_bc_raw()); } // 2-byte branch offset from next pc int next_get_dest() const { assert(_pc < _end, ""); - return next_bci() + next_bytecode()->get_offset_s2(Bytecodes::_ifeq); + return next_bci() + next_bytecode().get_offset_s2(Bytecodes::_ifeq); } // 4-byte branch offset from current pc int get_far_dest() const { - return cur_bci() + bytecode()->get_offset_s4(cur_bc_raw()); + return cur_bci() + bytecode().get_offset_s4(cur_bc_raw()); } // For a lookup or switch table, return target destination @@ -407,4 +407,11 @@ } }; + + +// Implementation for declarations in bytecode.hpp +Bytecode::Bytecode(const ciBytecodeStream* stream, address bcp): _bcp(bcp != NULL ? bcp : stream->cur_bcp()), _code(Bytecodes::code_at(NULL, addr_at(0))) {} +Bytecode_lookupswitch::Bytecode_lookupswitch(const ciBytecodeStream* stream): Bytecode(stream) { verify(); } +Bytecode_tableswitch::Bytecode_tableswitch(const ciBytecodeStream* stream): Bytecode(stream) { verify(); } + #endif // SHARE_VM_CI_CISTREAMS_HPP diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/ci/ciTypeFlow.cpp --- a/hotspot/src/share/vm/ci/ciTypeFlow.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/ci/ciTypeFlow.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1698,18 +1698,17 @@ break; case Bytecodes::_tableswitch: { - Bytecode_tableswitch *tableswitch = - Bytecode_tableswitch_at(str->cur_bcp()); + Bytecode_tableswitch tableswitch(str); - int len = tableswitch->length(); + int len = tableswitch.length(); _successors = new (arena) GrowableArray(arena, len+1, 0, NULL); - int bci = current_bci + tableswitch->default_offset(); + int bci = current_bci + tableswitch.default_offset(); Block* block = analyzer->block_at(bci, jsrs); assert(_successors->length() == SWITCH_DEFAULT, ""); _successors->append(block); while (--len >= 0) { - int bci = current_bci + tableswitch->dest_offset_at(len); + int bci = current_bci + tableswitch.dest_offset_at(len); block = analyzer->block_at(bci, jsrs); assert(_successors->length() >= SWITCH_CASES, ""); _successors->append_if_missing(block); @@ -1718,19 +1717,18 @@ } case Bytecodes::_lookupswitch: { - Bytecode_lookupswitch *lookupswitch = - Bytecode_lookupswitch_at(str->cur_bcp()); + Bytecode_lookupswitch lookupswitch(str); - int npairs = lookupswitch->number_of_pairs(); + int npairs = lookupswitch.number_of_pairs(); _successors = new (arena) GrowableArray(arena, npairs+1, 0, NULL); - int bci = current_bci + lookupswitch->default_offset(); + int bci = current_bci + lookupswitch.default_offset(); Block* block = analyzer->block_at(bci, jsrs); assert(_successors->length() == SWITCH_DEFAULT, ""); _successors->append(block); while(--npairs >= 0) { - LookupswitchPair *pair = lookupswitch->pair_at(npairs); - int bci = current_bci + pair->offset(); + LookupswitchPair pair = lookupswitch.pair_at(npairs); + int bci = current_bci + pair.offset(); Block* block = analyzer->block_at(bci, jsrs); assert(_successors->length() >= SWITCH_CASES, ""); _successors->append_if_missing(block); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/classfile/classLoader.cpp --- a/hotspot/src/share/vm/classfile/classLoader.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/classfile/classLoader.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1382,3 +1382,61 @@ } #endif //PRODUCT + +// Please keep following two functions at end of this file. With them placed at top or in middle of the file, +// they could get inlined by agressive compiler, an unknown trick, see bug 6966589. +void PerfClassTraceTime::initialize() { + if (!UsePerfData) return; + + if (_eventp != NULL) { + // increment the event counter + _eventp->inc(); + } + + // stop the current active thread-local timer to measure inclusive time + _prev_active_event = -1; + for (int i=0; i < EVENT_TYPE_COUNT; i++) { + if (_timers[i].is_active()) { + assert(_prev_active_event == -1, "should have only one active timer"); + _prev_active_event = i; + _timers[i].stop(); + } + } + + if (_recursion_counters == NULL || (_recursion_counters[_event_type])++ == 0) { + // start the inclusive timer if not recursively called + _t.start(); + } + + // start thread-local timer of the given event type + if (!_timers[_event_type].is_active()) { + _timers[_event_type].start(); + } +} + +PerfClassTraceTime::~PerfClassTraceTime() { + if (!UsePerfData) return; + + // stop the thread-local timer as the event completes + // and resume the thread-local timer of the event next on the stack + _timers[_event_type].stop(); + jlong selftime = _timers[_event_type].ticks(); + + if (_prev_active_event >= 0) { + _timers[_prev_active_event].start(); + } + + if (_recursion_counters != NULL && --(_recursion_counters[_event_type]) > 0) return; + + // increment the counters only on the leaf call + _t.stop(); + _timep->inc(_t.ticks()); + if (_selftimep != NULL) { + _selftimep->inc(selftime); + } + // add all class loading related event selftime to the accumulated time counter + ClassLoader::perf_accumulated_time()->inc(selftime); + + // reset the timer + _timers[_event_type].reset(); +} diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/classfile/classLoader.hpp --- a/hotspot/src/share/vm/classfile/classLoader.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/classfile/classLoader.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -356,111 +356,57 @@ // (i.e. only one event type) are active at a time even multiple PerfClassTraceTime // instances have been created as multiple events are happening. class PerfClassTraceTime { - public: - enum { - CLASS_LOAD = 0, - PARSE_CLASS = 1, - CLASS_LINK = 2, - CLASS_VERIFY = 3, - CLASS_CLINIT = 4, - DEFINE_CLASS = 5, - EVENT_TYPE_COUNT = 6 - }; - protected: - // _t tracks time from initialization to destruction of this timer instance - // including time for all other event types, and recursive calls of this type. - // When a timer is called recursively, the elapsedTimer _t would not be used. - elapsedTimer _t; - PerfLongCounter* _timep; - PerfLongCounter* _selftimep; - PerfLongCounter* _eventp; - // pointer to thread-local recursion counter and timer array - // The thread_local timers track cumulative time for specific event types - // exclusive of time for other event types, but including recursive calls - // of the same type. - int* _recursion_counters; - elapsedTimer* _timers; - int _event_type; - int _prev_active_event; - - public: - - inline PerfClassTraceTime(PerfLongCounter* timep, /* counter incremented with inclusive time */ - PerfLongCounter* selftimep, /* counter incremented with exclusive time */ - PerfLongCounter* eventp, /* event counter */ - int* recursion_counters, /* thread-local recursion counter array */ - elapsedTimer* timers, /* thread-local timer array */ - int type /* event type */ ) : - _timep(timep), _selftimep(selftimep), _eventp(eventp), _recursion_counters(recursion_counters), _timers(timers), _event_type(type) { - initialize(); - } - - inline PerfClassTraceTime(PerfLongCounter* timep, /* counter incremented with inclusive time */ - elapsedTimer* timers, /* thread-local timer array */ - int type /* event type */ ) : - _timep(timep), _selftimep(NULL), _eventp(NULL), _recursion_counters(NULL), _timers(timers), _event_type(type) { - initialize(); - } - - void initialize() { - if (!UsePerfData) return; + public: + enum { + CLASS_LOAD = 0, + PARSE_CLASS = 1, + CLASS_LINK = 2, + CLASS_VERIFY = 3, + CLASS_CLINIT = 4, + DEFINE_CLASS = 5, + EVENT_TYPE_COUNT = 6 + }; + protected: + // _t tracks time from initialization to destruction of this timer instance + // including time for all other event types, and recursive calls of this type. + // When a timer is called recursively, the elapsedTimer _t would not be used. + elapsedTimer _t; + PerfLongCounter* _timep; + PerfLongCounter* _selftimep; + PerfLongCounter* _eventp; + // pointer to thread-local recursion counter and timer array + // The thread_local timers track cumulative time for specific event types + // exclusive of time for other event types, but including recursive calls + // of the same type. + int* _recursion_counters; + elapsedTimer* _timers; + int _event_type; + int _prev_active_event; - if (_eventp != NULL) { - // increment the event counter - _eventp->inc(); - } + public: - // stop the current active thread-local timer to measure inclusive time - _prev_active_event = -1; - for (int i=0; i < EVENT_TYPE_COUNT; i++) { - if (_timers[i].is_active()) { - assert(_prev_active_event == -1, "should have only one active timer"); - _prev_active_event = i; - _timers[i].stop(); - } - } - - if (_recursion_counters == NULL || (_recursion_counters[_event_type])++ == 0) { - // start the inclusive timer if not recursively called - _t.start(); - } - - // start thread-local timer of the given event type - if (!_timers[_event_type].is_active()) { - _timers[_event_type].start(); - } - } + inline PerfClassTraceTime(PerfLongCounter* timep, /* counter incremented with inclusive time */ + PerfLongCounter* selftimep, /* counter incremented with exclusive time */ + PerfLongCounter* eventp, /* event counter */ + int* recursion_counters, /* thread-local recursion counter array */ + elapsedTimer* timers, /* thread-local timer array */ + int type /* event type */ ) : + _timep(timep), _selftimep(selftimep), _eventp(eventp), _recursion_counters(recursion_counters), _timers(timers), _event_type(type) { + initialize(); + } - inline void suspend() { _t.stop(); _timers[_event_type].stop(); } - inline void resume() { _t.start(); _timers[_event_type].start(); } - - ~PerfClassTraceTime() { - if (!UsePerfData) return; - - // stop the thread-local timer as the event completes - // and resume the thread-local timer of the event next on the stack - _timers[_event_type].stop(); - jlong selftime = _timers[_event_type].ticks(); - - if (_prev_active_event >= 0) { - _timers[_prev_active_event].start(); - } + inline PerfClassTraceTime(PerfLongCounter* timep, /* counter incremented with inclusive time */ + elapsedTimer* timers, /* thread-local timer array */ + int type /* event type */ ) : + _timep(timep), _selftimep(NULL), _eventp(NULL), _recursion_counters(NULL), _timers(timers), _event_type(type) { + initialize(); + } - if (_recursion_counters != NULL && --(_recursion_counters[_event_type]) > 0) return; + inline void suspend() { _t.stop(); _timers[_event_type].stop(); } + inline void resume() { _t.start(); _timers[_event_type].start(); } - // increment the counters only on the leaf call - _t.stop(); - _timep->inc(_t.ticks()); - if (_selftimep != NULL) { - _selftimep->inc(selftime); - } - // add all class loading related event selftime to the accumulated time counter - ClassLoader::perf_accumulated_time()->inc(selftime); - - // reset the timer - _timers[_event_type].reset(); - } + ~PerfClassTraceTime(); + void initialize(); }; - #endif // SHARE_VM_CLASSFILE_CLASSLOADER_HPP diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/classfile/systemDictionary.hpp --- a/hotspot/src/share/vm/classfile/systemDictionary.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/classfile/systemDictionary.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -172,6 +172,8 @@ \ template(sun_jkernel_DownloadManager_klass, sun_jkernel_DownloadManager, Opt_Kernel) \ \ + template(sun_misc_PostVMInitHook_klass, sun_misc_PostVMInitHook, Opt) \ + \ /* Preload boxing klasses */ \ template(Boolean_klass, java_lang_Boolean, Pre) \ template(Character_klass, java_lang_Character, Pre) \ diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/classfile/vmSymbols.hpp --- a/hotspot/src/share/vm/classfile/vmSymbols.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/classfile/vmSymbols.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -111,6 +111,7 @@ template(sun_jkernel_DownloadManager, "sun/jkernel/DownloadManager") \ template(getBootClassPathEntryForClass_name, "getBootClassPathEntryForClass") \ template(setBootClassLoaderHook_name, "setBootClassLoaderHook") \ + template(sun_misc_PostVMInitHook, "sun/misc/PostVMInitHook") \ \ /* class file format tags */ \ template(tag_source_file, "SourceFile") \ diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/code/nmethod.cpp --- a/hotspot/src/share/vm/code/nmethod.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/code/nmethod.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1863,9 +1863,9 @@ #ifndef SHARK if (!method()->is_native()) { SimpleScopeDesc ssd(this, fr.pc()); - Bytecode_invoke* call = Bytecode_invoke_at(ssd.method(), ssd.bci()); - bool has_receiver = call->has_receiver(); - symbolOop signature = call->signature(); + Bytecode_invoke call(ssd.method(), ssd.bci()); + bool has_receiver = call.has_receiver(); + symbolOop signature = call.signature(); fr.oops_compiled_arguments_do(signature, has_receiver, reg_map, f); } #endif // !SHARK @@ -2698,8 +2698,7 @@ } else if (sd->method()->is_native()) { st->print("method is native"); } else { - address bcp = sd->method()->bcp_from(sd->bci()); - Bytecodes::Code bc = Bytecodes::java_code_at(bcp); + Bytecodes::Code bc = sd->method()->java_code_at(sd->bci()); st->print(";*%s", Bytecodes::name(bc)); switch (bc) { case Bytecodes::_invokevirtual: @@ -2707,10 +2706,10 @@ case Bytecodes::_invokestatic: case Bytecodes::_invokeinterface: { - Bytecode_invoke* invoke = Bytecode_invoke_at(sd->method(), sd->bci()); + Bytecode_invoke invoke(sd->method(), sd->bci()); st->print(" "); - if (invoke->name() != NULL) - invoke->name()->print_symbol_on(st); + if (invoke.name() != NULL) + invoke.name()->print_symbol_on(st); else st->print(""); break; @@ -2720,10 +2719,10 @@ case Bytecodes::_getstatic: case Bytecodes::_putstatic: { - Bytecode_field* field = Bytecode_field_at(sd->method(), sd->bci()); + Bytecode_field field(sd->method(), sd->bci()); st->print(" "); - if (field->name() != NULL) - field->name()->print_symbol_on(st); + if (field.name() != NULL) + field.name()->print_symbol_on(st); else st->print(""); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/compiler/methodLiveness.cpp --- a/hotspot/src/share/vm/compiler/methodLiveness.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/compiler/methodLiveness.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -286,16 +286,15 @@ break; case Bytecodes::_tableswitch: { - Bytecode_tableswitch *tableswitch = - Bytecode_tableswitch_at(bytes.cur_bcp()); + Bytecode_tableswitch tableswitch(&bytes); - int len = tableswitch->length(); + int len = tableswitch.length(); - dest = _block_map->at(bci + tableswitch->default_offset()); + dest = _block_map->at(bci + tableswitch.default_offset()); assert(dest != NULL, "branch desination must start a block."); dest->add_normal_predecessor(current_block); while (--len >= 0) { - dest = _block_map->at(bci + tableswitch->dest_offset_at(len)); + dest = _block_map->at(bci + tableswitch.dest_offset_at(len)); assert(dest != NULL, "branch desination must start a block."); dest->add_normal_predecessor(current_block); } @@ -304,17 +303,16 @@ case Bytecodes::_lookupswitch: { - Bytecode_lookupswitch *lookupswitch = - Bytecode_lookupswitch_at(bytes.cur_bcp()); + Bytecode_lookupswitch lookupswitch(&bytes); - int npairs = lookupswitch->number_of_pairs(); + int npairs = lookupswitch.number_of_pairs(); - dest = _block_map->at(bci + lookupswitch->default_offset()); + dest = _block_map->at(bci + lookupswitch.default_offset()); assert(dest != NULL, "branch desination must start a block."); dest->add_normal_predecessor(current_block); while(--npairs >= 0) { - LookupswitchPair *pair = lookupswitch->pair_at(npairs); - dest = _block_map->at( bci + pair->offset()); + LookupswitchPair pair = lookupswitch.pair_at(npairs); + dest = _block_map->at( bci + pair.offset()); assert(dest != NULL, "branch desination must start a block."); dest->add_normal_predecessor(current_block); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp --- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -3478,6 +3478,7 @@ assert(_collectorState == InitialMarking, "Wrong collector state"); check_correct_thread_executing(); TraceCMSMemoryManagerStats tms(_collectorState); + ReferenceProcessor* rp = ref_processor(); SpecializationStats::clear(); assert(_restart_addr == NULL, "Control point invariant"); @@ -4978,6 +4979,7 @@ if (should_unload_classes()) { CodeCache::gc_epilogue(); } + JvmtiExport::gc_epilogue(); // If we encountered any (marking stack / work queue) overflow // events during the current CMS cycle, take appropriate @@ -5940,11 +5942,6 @@ } rp->verify_no_references_recorded(); assert(!rp->discovery_enabled(), "should have been disabled"); - - // JVMTI object tagging is based on JNI weak refs. If any of these - // refs were cleared then JVMTI needs to update its maps and - // maybe post ObjectFrees to agents. - JvmtiExport::cms_ref_processing_epilogue(); } #ifndef PRODUCT @@ -6305,6 +6302,7 @@ switch (op) { case CMS_op_checkpointRootsInitial: { + SvcGCMarker sgcm(SvcGCMarker::OTHER); checkpointRootsInitial(true); // asynch if (PrintGC) { _cmsGen->printOccupancy("initial-mark"); @@ -6312,6 +6310,7 @@ break; } case CMS_op_checkpointRootsFinal: { + SvcGCMarker sgcm(SvcGCMarker::OTHER); checkpointRootsFinal(true, // asynch false, // !clear_all_soft_refs false); // !init_mark_was_synchronous @@ -7881,25 +7880,23 @@ } // We need this destructor to reclaim any space at the end -// of the space, which do_blk below may not have added back to -// the free lists. [basically dealing with the "fringe effect"] +// of the space, which do_blk below may not yet have added back to +// the free lists. SweepClosure::~SweepClosure() { assert_lock_strong(_freelistLock); - // this should be treated as the end of a free run if any - // The current free range should be returned to the free lists - // as one coalesced chunk. + assert(_limit >= _sp->bottom() && _limit <= _sp->end(), + "sweep _limit out of bounds"); + // Flush any remaining coterminal free run as a single + // coalesced chunk to the appropriate free list. if (inFreeRange()) { - flushCurFreeChunk(freeFinger(), - pointer_delta(_limit, freeFinger())); - assert(freeFinger() < _limit, "the finger pointeth off base"); + assert(freeFinger() < _limit, "freeFinger points too high"); + flush_cur_free_chunk(freeFinger(), pointer_delta(_limit, freeFinger())); if (CMSTraceSweeper) { - gclog_or_tty->print("destructor:"); - gclog_or_tty->print("Sweep:put_free_blk 0x%x ("SIZE_FORMAT") " - "[coalesced:"SIZE_FORMAT"]\n", - freeFinger(), pointer_delta(_limit, freeFinger()), - lastFreeRangeCoalesced()); - } - } + gclog_or_tty->print("Sweep: last chunk: "); + gclog_or_tty->print("put_free_blk 0x%x ("SIZE_FORMAT") [coalesced:"SIZE_FORMAT"]\n", + freeFinger(), pointer_delta(_limit, freeFinger()), lastFreeRangeCoalesced()); + } + } // else nothing to flush NOT_PRODUCT( if (Verbose && PrintGC) { gclog_or_tty->print("Collected "SIZE_FORMAT" objects, " @@ -7936,9 +7933,8 @@ void SweepClosure::initialize_free_range(HeapWord* freeFinger, bool freeRangeInFreeLists) { if (CMSTraceSweeper) { - gclog_or_tty->print("---- Start free range 0x%x with free block [%d] (%d)\n", - freeFinger, _sp->block_size(freeFinger), - freeRangeInFreeLists); + gclog_or_tty->print("---- Start free range at 0x%x with free block (%d)\n", + freeFinger, freeRangeInFreeLists); } assert(!inFreeRange(), "Trampling existing free range"); set_inFreeRange(true); @@ -7993,21 +7989,36 @@ // may have caused us to coalesce the block ending at the address _limit // with a newly expanded chunk (this happens when _limit was set to the // previous _end of the space), so we may have stepped past _limit; see CR 6977970. - if (addr >= _limit) { // we have swept up to or past the limit, do nothing more + if (addr >= _limit) { // we have swept up to or past the limit: finish up assert(_limit >= _sp->bottom() && _limit <= _sp->end(), "sweep _limit out of bounds"); assert(addr < _sp->end(), "addr out of bounds"); - // help the closure application finish + // Flush any remaining coterminal free run as a single + // coalesced chunk to the appropriate free list. + if (inFreeRange()) { + assert(freeFinger() < _limit, "finger points too high"); + flush_cur_free_chunk(freeFinger(), + pointer_delta(addr, freeFinger())); + if (CMSTraceSweeper) { + gclog_or_tty->print("Sweep: last chunk: "); + gclog_or_tty->print("put_free_blk 0x%x ("SIZE_FORMAT") " + "[coalesced:"SIZE_FORMAT"]\n", + freeFinger(), pointer_delta(addr, freeFinger()), + lastFreeRangeCoalesced()); + } + } + + // help the iterator loop finish return pointer_delta(_sp->end(), addr); } + assert(addr < _limit, "sweep invariant"); - // check if we should yield do_yield_check(addr); if (fc->isFree()) { // Chunk that is already free res = fc->size(); - doAlreadyFreeChunk(fc); + do_already_free_chunk(fc); debug_only(_sp->verifyFreeLists()); assert(res == fc->size(), "Don't expect the size to change"); NOT_PRODUCT( @@ -8017,7 +8028,7 @@ NOT_PRODUCT(_last_fc = fc;) } else if (!_bitMap->isMarked(addr)) { // Chunk is fresh garbage - res = doGarbageChunk(fc); + res = do_garbage_chunk(fc); debug_only(_sp->verifyFreeLists()); NOT_PRODUCT( _numObjectsFreed++; @@ -8025,7 +8036,7 @@ ) } else { // Chunk that is alive. - res = doLiveChunk(fc); + res = do_live_chunk(fc); debug_only(_sp->verifyFreeLists()); NOT_PRODUCT( _numObjectsLive++; @@ -8078,7 +8089,7 @@ // to a free list which may be overpopulated. // -void SweepClosure::doAlreadyFreeChunk(FreeChunk* fc) { +void SweepClosure::do_already_free_chunk(FreeChunk* fc) { size_t size = fc->size(); // Chunks that cannot be coalesced are not in the // free lists. @@ -8094,23 +8105,23 @@ // addr and purported end of this block. _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size); - // Some chunks cannot be coalesced in under any circumstances. + // Some chunks cannot be coalesced under any circumstances. // See the definition of cantCoalesce(). if (!fc->cantCoalesce()) { // This chunk can potentially be coalesced. if (_sp->adaptive_freelists()) { // All the work is done in - doPostIsFreeOrGarbageChunk(fc, size); + do_post_free_or_garbage_chunk(fc, size); } else { // Not adaptive free lists // this is a free chunk that can potentially be coalesced by the sweeper; if (!inFreeRange()) { // if the next chunk is a free block that can't be coalesced // it doesn't make sense to remove this chunk from the free lists FreeChunk* nextChunk = (FreeChunk*)(addr + size); - assert((HeapWord*)nextChunk <= _limit, "sweep invariant"); - if ((HeapWord*)nextChunk < _limit && // there's a next chunk... - nextChunk->isFree() && // which is free... - nextChunk->cantCoalesce()) { // ... but cant be coalesced + assert((HeapWord*)nextChunk <= _sp->end(), "Chunk size out of bounds?"); + if ((HeapWord*)nextChunk < _sp->end() && // There is another free chunk to the right ... + nextChunk->isFree() && // ... which is free... + nextChunk->cantCoalesce()) { // ... but can't be coalesced // nothing to do } else { // Potentially the start of a new free range: @@ -8156,14 +8167,14 @@ // as the end of a free run if any if (inFreeRange()) { // we kicked some butt; time to pick up the garbage - assert(freeFinger() < addr, "the finger pointeth off base"); - flushCurFreeChunk(freeFinger(), pointer_delta(addr, freeFinger())); + assert(freeFinger() < addr, "freeFinger points too high"); + flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger())); } // else, nothing to do, just continue } } -size_t SweepClosure::doGarbageChunk(FreeChunk* fc) { +size_t SweepClosure::do_garbage_chunk(FreeChunk* fc) { // This is a chunk of garbage. It is not in any free list. // Add it to a free list or let it possibly be coalesced into // a larger chunk. @@ -8175,7 +8186,7 @@ // addr and purported end of just dead object. _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size); - doPostIsFreeOrGarbageChunk(fc, size); + do_post_free_or_garbage_chunk(fc, size); } else { if (!inFreeRange()) { // start of a new free range @@ -8214,35 +8225,16 @@ return size; } -size_t SweepClosure::doLiveChunk(FreeChunk* fc) { +size_t SweepClosure::do_live_chunk(FreeChunk* fc) { HeapWord* addr = (HeapWord*) fc; // The sweeper has just found a live object. Return any accumulated // left hand chunk to the free lists. if (inFreeRange()) { - if (_sp->adaptive_freelists()) { - flushCurFreeChunk(freeFinger(), - pointer_delta(addr, freeFinger())); - } else { // not adaptive freelists - set_inFreeRange(false); - // Add the free range back to the free list if it is not already - // there. - if (!freeRangeInFreeLists()) { - assert(freeFinger() < addr, "the finger pointeth off base"); - if (CMSTraceSweeper) { - gclog_or_tty->print("Sweep:put_free_blk 0x%x (%d) " - "[coalesced:%d]\n", - freeFinger(), pointer_delta(addr, freeFinger()), - lastFreeRangeCoalesced()); - } - _sp->addChunkAndRepairOffsetTable(freeFinger(), - pointer_delta(addr, freeFinger()), lastFreeRangeCoalesced()); - } - } - } - - // Common code path for original and adaptive free lists. - - // this object is live: we'd normally expect this to be + assert(freeFinger() < addr, "freeFinger points too high"); + flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger())); + } + + // This object is live: we'd normally expect this to be // an oop, and like to assert the following: // assert(oop(addr)->is_oop(), "live block should be an oop"); // However, as we commented above, this may be an object whose @@ -8257,7 +8249,7 @@ assert(size == CompactibleFreeListSpace::adjustObjectSize(size), "alignment problem"); - #ifdef DEBUG +#ifdef DEBUG if (oop(addr)->klass_or_null() != NULL && ( !_collector->should_unload_classes() || (oop(addr)->is_parsable()) && @@ -8271,7 +8263,7 @@ CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size()), "P-mark and computed size do not agree"); } - #endif +#endif } else { // This should be an initialized object that's alive. @@ -8298,18 +8290,16 @@ return size; } -void SweepClosure::doPostIsFreeOrGarbageChunk(FreeChunk* fc, - size_t chunkSize) { - // doPostIsFreeOrGarbageChunk() should only be called in the smart allocation - // scheme. +void SweepClosure::do_post_free_or_garbage_chunk(FreeChunk* fc, + size_t chunkSize) { + // do_post_free_or_garbage_chunk() should only be called in the case + // of the adaptive free list allocator. bool fcInFreeLists = fc->isFree(); assert(_sp->adaptive_freelists(), "Should only be used in this case."); assert((HeapWord*)fc <= _limit, "sweep invariant"); if (CMSTestInFreeList && fcInFreeLists) { - assert(_sp->verifyChunkInFreeLists(fc), - "free chunk is not in free lists"); - } - + assert(_sp->verifyChunkInFreeLists(fc), "free chunk is not in free lists"); + } if (CMSTraceSweeper) { gclog_or_tty->print_cr(" -- pick up another chunk at 0x%x (%d)", fc, chunkSize); @@ -8382,20 +8372,21 @@ if (inFreeRange()) { // In a free range but cannot coalesce with the right hand chunk. // Put the current free range into the free lists. - flushCurFreeChunk(freeFinger(), - pointer_delta(addr, freeFinger())); + flush_cur_free_chunk(freeFinger(), + pointer_delta(addr, freeFinger())); } // Set up for new free range. Pass along whether the right hand // chunk is in the free lists. initialize_free_range((HeapWord*)fc, fcInFreeLists); } } -void SweepClosure::flushCurFreeChunk(HeapWord* chunk, size_t size) { + +void SweepClosure::flush_cur_free_chunk(HeapWord* chunk, size_t size) { assert(inFreeRange(), "Should only be called if currently in a free range."); assert(size > 0, "A zero sized chunk cannot be added to the free lists."); if (!freeRangeInFreeLists()) { - if(CMSTestInFreeList) { + if (CMSTestInFreeList) { FreeChunk* fc = (FreeChunk*) chunk; fc->setSize(size); assert(!_sp->verifyChunkInFreeLists(fc), @@ -8430,7 +8421,7 @@ // chunk just flushed, they will need to wait for the next // sweep to be coalesced. if (inFreeRange()) { - flushCurFreeChunk(freeFinger(), pointer_delta(addr, freeFinger())); + flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger())); } // First give up the locks, then yield, then re-lock. diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp --- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1701,7 +1701,9 @@ CMSCollector* _collector; // collector doing the work ConcurrentMarkSweepGeneration* _g; // Generation being swept CompactibleFreeListSpace* _sp; // Space being swept - HeapWord* _limit; + HeapWord* _limit;// the address at which the sweep should stop because + // we do not expect blocks eligible for sweeping past + // that address. Mutex* _freelistLock; // Free list lock (in space) CMSBitMap* _bitMap; // Marking bit map (in // generation) @@ -1745,14 +1747,13 @@ private: // Code that is common to a free chunk or garbage when // encountered during sweeping. - void doPostIsFreeOrGarbageChunk(FreeChunk *fc, - size_t chunkSize); + void do_post_free_or_garbage_chunk(FreeChunk *fc, size_t chunkSize); // Process a free chunk during sweeping. - void doAlreadyFreeChunk(FreeChunk *fc); + void do_already_free_chunk(FreeChunk *fc); // Process a garbage chunk during sweeping. - size_t doGarbageChunk(FreeChunk *fc); + size_t do_garbage_chunk(FreeChunk *fc); // Process a live chunk during sweeping. - size_t doLiveChunk(FreeChunk* fc); + size_t do_live_chunk(FreeChunk* fc); // Accessors. HeapWord* freeFinger() const { return _freeFinger; } @@ -1769,7 +1770,7 @@ // Initialize a free range. void initialize_free_range(HeapWord* freeFinger, bool freeRangeInFreeLists); // Return this chunk to the free lists. - void flushCurFreeChunk(HeapWord* chunk, size_t size); + void flush_cur_free_chunk(HeapWord* chunk, size_t size); // Check if we should yield and do so when necessary. inline void do_yield_check(HeapWord* addr); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/concurrentMark.cpp --- a/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,6 +31,7 @@ #include "gc_implementation/g1/g1RemSet.hpp" #include "gc_implementation/g1/heapRegionRemSet.hpp" #include "gc_implementation/g1/heapRegionSeq.inline.hpp" +#include "gc_implementation/shared/vmGCOperations.hpp" #include "memory/genOopClosures.inline.hpp" #include "memory/referencePolicy.hpp" #include "memory/resourceArea.hpp" @@ -457,6 +458,7 @@ _marking_task_overhead(1.0), _cleanup_sleep_factor(0.0), _cleanup_task_overhead(1.0), + _cleanup_list("Cleanup List"), _region_bm(max_regions, false /* in_resource_area*/), _card_bm((rs.size() + CardTableModRefBS::card_size - 1) >> CardTableModRefBS::card_shift, @@ -520,12 +522,6 @@ SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set(); satb_qs.set_buffer_size(G1SATBBufferSize); - int size = (int) MAX2(ParallelGCThreads, (size_t)1); - _par_cleanup_thread_state = NEW_C_HEAP_ARRAY(ParCleanupThreadState*, size); - for (int i = 0 ; i < size; i++) { - _par_cleanup_thread_state[i] = new ParCleanupThreadState; - } - _tasks = NEW_C_HEAP_ARRAY(CMTask*, _max_task_num); _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_task_num); @@ -710,11 +706,6 @@ } ConcurrentMark::~ConcurrentMark() { - int size = (int) MAX2(ParallelGCThreads, (size_t)1); - for (int i = 0; i < size; i++) delete _par_cleanup_thread_state[i]; - FREE_C_HEAP_ARRAY(ParCleanupThreadState*, - _par_cleanup_thread_state); - for (int i = 0; i < (int) _max_task_num; ++i) { delete _task_queues->queue(i); delete _tasks[i]; @@ -1142,6 +1133,8 @@ return; } + SvcGCMarker sgcm(SvcGCMarker::OTHER); + if (VerifyDuringGC) { HandleMark hm; // handle scope gclog_or_tty->print(" VerifyDuringGC:(before)"); @@ -1168,12 +1161,12 @@ if (G1TraceMarkStackOverflow) gclog_or_tty->print_cr("\nRemark led to restart for overflow."); } else { + SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); // We're done with marking. // This is the end of the marking cycle, we're expected all // threads to have SATB queues with active set to true. - JavaThread::satb_mark_queue_set().set_active_all_threads( - false, /* new active value */ - true /* expected_active */); + satb_mq_set.set_active_all_threads(false, /* new active value */ + true /* expected_active */); if (VerifyDuringGC) { HandleMark hm; // handle scope @@ -1507,21 +1500,20 @@ size_t _max_live_bytes; size_t _regions_claimed; size_t _freed_bytes; - size_t _cleared_h_regions; - size_t _freed_regions; - UncleanRegionList* _unclean_region_list; + FreeRegionList _local_cleanup_list; + HumongousRegionSet _humongous_proxy_set; double _claimed_region_time; double _max_region_time; public: G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1, - UncleanRegionList* list, int worker_num); size_t freed_bytes() { return _freed_bytes; } - size_t cleared_h_regions() { return _cleared_h_regions; } - size_t freed_regions() { return _freed_regions; } - UncleanRegionList* unclean_region_list() { - return _unclean_region_list; + FreeRegionList* local_cleanup_list() { + return &_local_cleanup_list; + } + HumongousRegionSet* humongous_proxy_set() { + return &_humongous_proxy_set; } bool doHeapRegion(HeapRegion *r); @@ -1534,25 +1526,22 @@ class G1ParNoteEndTask: public AbstractGangTask { friend class G1NoteEndOfConcMarkClosure; + protected: G1CollectedHeap* _g1h; size_t _max_live_bytes; size_t _freed_bytes; - ConcurrentMark::ParCleanupThreadState** _par_cleanup_thread_state; + FreeRegionList* _cleanup_list; + public: G1ParNoteEndTask(G1CollectedHeap* g1h, - ConcurrentMark::ParCleanupThreadState** - par_cleanup_thread_state) : + FreeRegionList* cleanup_list) : AbstractGangTask("G1 note end"), _g1h(g1h), - _max_live_bytes(0), _freed_bytes(0), - _par_cleanup_thread_state(par_cleanup_thread_state) - {} + _max_live_bytes(0), _freed_bytes(0), _cleanup_list(cleanup_list) { } void work(int i) { double start = os::elapsedTime(); - G1NoteEndOfConcMarkClosure g1_note_end(_g1h, - &_par_cleanup_thread_state[i]->list, - i); + G1NoteEndOfConcMarkClosure g1_note_end(_g1h, i); if (G1CollectedHeap::use_parallel_gc_threads()) { _g1h->heap_region_par_iterate_chunked(&g1_note_end, i, HeapRegion::NoteEndClaimValue); @@ -1561,14 +1550,18 @@ } assert(g1_note_end.complete(), "Shouldn't have yielded!"); - // Now finish up freeing the current thread's regions. - _g1h->finish_free_region_work(g1_note_end.freed_bytes(), - g1_note_end.cleared_h_regions(), - 0, NULL); + // Now update the lists + _g1h->update_sets_after_freeing_regions(g1_note_end.freed_bytes(), + NULL /* free_list */, + g1_note_end.humongous_proxy_set(), + true /* par */); { MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); _max_live_bytes += g1_note_end.max_live_bytes(); _freed_bytes += g1_note_end.freed_bytes(); + + _cleanup_list->add_as_tail(g1_note_end.local_cleanup_list()); + assert(g1_note_end.local_cleanup_list()->is_empty(), "post-condition"); } double end = os::elapsedTime(); if (G1PrintParCleanupStats) { @@ -1609,30 +1602,28 @@ G1NoteEndOfConcMarkClosure:: G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1, - UncleanRegionList* list, int worker_num) : _g1(g1), _worker_num(worker_num), _max_live_bytes(0), _regions_claimed(0), - _freed_bytes(0), _cleared_h_regions(0), _freed_regions(0), + _freed_bytes(0), _claimed_region_time(0.0), _max_region_time(0.0), - _unclean_region_list(list) -{} - -bool G1NoteEndOfConcMarkClosure::doHeapRegion(HeapRegion *r) { + _local_cleanup_list("Local Cleanup List"), + _humongous_proxy_set("Local Cleanup Humongous Proxy Set") { } + +bool G1NoteEndOfConcMarkClosure::doHeapRegion(HeapRegion *hr) { // We use a claim value of zero here because all regions // were claimed with value 1 in the FinalCount task. - r->reset_gc_time_stamp(); - if (!r->continuesHumongous()) { + hr->reset_gc_time_stamp(); + if (!hr->continuesHumongous()) { double start = os::elapsedTime(); _regions_claimed++; - r->note_end_of_marking(); - _max_live_bytes += r->max_live_bytes(); - _g1->free_region_if_totally_empty_work(r, - _freed_bytes, - _cleared_h_regions, - _freed_regions, - _unclean_region_list, - true /*par*/); + hr->note_end_of_marking(); + _max_live_bytes += hr->max_live_bytes(); + _g1->free_region_if_totally_empty(hr, + &_freed_bytes, + &_local_cleanup_list, + &_humongous_proxy_set, + true /* par */); double region_time = (os::elapsedTime() - start); _claimed_region_time += region_time; if (region_time > _max_region_time) _max_region_time = region_time; @@ -1652,6 +1643,8 @@ return; } + g1h->verify_region_sets_optional(); + if (VerifyDuringGC) { HandleMark hm; // handle scope gclog_or_tty->print(" VerifyDuringGC:(before)"); @@ -1716,7 +1709,7 @@ // Note end of marking in all heap regions. double note_end_start = os::elapsedTime(); - G1ParNoteEndTask g1_par_note_end_task(g1h, _par_cleanup_thread_state); + G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list); if (G1CollectedHeap::use_parallel_gc_threads()) { int n_workers = g1h->workers()->total_workers(); g1h->set_par_threads(n_workers); @@ -1728,9 +1721,14 @@ } else { g1_par_note_end_task.work(0); } - g1h->set_unclean_regions_coming(true); + + if (!cleanup_list_is_empty()) { + // The cleanup list is not empty, so we'll have to process it + // concurrently. Notify anyone else that might be wanting free + // regions that there will be more free regions coming soon. + g1h->set_free_regions_coming(); + } double note_end_end = os::elapsedTime(); - // Tell the mutators that there might be unclean regions coming... if (G1PrintParCleanupStats) { gclog_or_tty->print_cr(" note end of marking: %8.3f ms.", (note_end_end - note_end_start)*1000.0); @@ -1796,33 +1794,63 @@ /* silent */ false, /* prev marking */ true); } + + g1h->verify_region_sets_optional(); } void ConcurrentMark::completeCleanup() { - // A full collection intervened. if (has_aborted()) return; - int first = 0; - int last = (int)MAX2(ParallelGCThreads, (size_t)1); - for (int t = 0; t < last; t++) { - UncleanRegionList* list = &_par_cleanup_thread_state[t]->list; - assert(list->well_formed(), "Inv"); - HeapRegion* hd = list->hd(); - while (hd != NULL) { - // Now finish up the other stuff. - hd->rem_set()->clear(); - HeapRegion* next_hd = hd->next_from_unclean_list(); - (void)list->pop(); - assert(list->hd() == next_hd, "how not?"); - _g1h->put_region_on_unclean_list(hd); - if (!hd->isHumongous()) { - // Add this to the _free_regions count by 1. - _g1h->finish_free_region_work(0, 0, 1, NULL); + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + _cleanup_list.verify_optional(); + FreeRegionList local_free_list("Local Cleanup List"); + + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : " + "cleanup list has "SIZE_FORMAT" entries", + _cleanup_list.length()); + } + + // Noone else should be accessing the _cleanup_list at this point, + // so it's not necessary to take any locks + while (!_cleanup_list.is_empty()) { + HeapRegion* hr = _cleanup_list.remove_head(); + assert(hr != NULL, "the list was not empty"); + hr->rem_set()->clear(); + local_free_list.add_as_tail(hr); + + // Instead of adding one region at a time to the secondary_free_list, + // we accumulate them in the local list and move them a few at a + // time. This also cuts down on the number of notify_all() calls + // we do during this process. We'll also append the local list when + // _cleanup_list is empty (which means we just removed the last + // region from the _cleanup_list). + if ((local_free_list.length() % G1SecondaryFreeListAppendLength == 0) || + _cleanup_list.is_empty()) { + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : " + "appending "SIZE_FORMAT" entries to the " + "secondary_free_list, clean list still has " + SIZE_FORMAT" entries", + local_free_list.length(), + _cleanup_list.length()); } - hd = list->hd(); - assert(hd == next_hd, "how not?"); + + { + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + g1h->secondary_free_list_add_as_tail(&local_free_list); + SecondaryFreeList_lock->notify_all(); + } + + if (G1StressConcRegionFreeing) { + for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) { + os::sleep(Thread::current(), (jlong) 1, false); + } + } } } + assert(local_free_list.is_empty(), "post-condition"); } bool G1CMIsAliveClosure::do_object_b(oop obj) { @@ -2894,9 +2922,9 @@ virtual void do_oop( oop* p) { do_oop_work(p); } template void do_oop_work(T* p) { - assert(_g1h->is_in_g1_reserved((HeapWord*) p), "invariant"); - assert(!_g1h->heap_region_containing((HeapWord*) p)->is_on_free_list(), - "invariant"); + assert( _g1h->is_in_g1_reserved((HeapWord*) p), "invariant"); + assert(!_g1h->is_on_free_list( + _g1h->heap_region_containing((HeapWord*) p)), "invariant"); oop obj = oopDesc::load_decode_heap_oop(p); if (_cm->verbose_high()) @@ -3116,8 +3144,8 @@ void CMTask::push(oop obj) { HeapWord* objAddr = (HeapWord*) obj; assert(_g1h->is_in_g1_reserved(objAddr), "invariant"); - assert(!_g1h->heap_region_containing(objAddr)->is_on_free_list(), - "invariant"); + assert(!_g1h->is_on_free_list( + _g1h->heap_region_containing((HeapWord*) objAddr)), "invariant"); assert(!_g1h->is_obj_ill(obj), "invariant"); assert(_nextMarkBitMap->isMarked(objAddr), "invariant"); @@ -3362,8 +3390,8 @@ (void*) obj); assert(_g1h->is_in_g1_reserved((HeapWord*) obj), "invariant" ); - assert(!_g1h->heap_region_containing(obj)->is_on_free_list(), - "invariant"); + assert(!_g1h->is_on_free_list( + _g1h->heap_region_containing((HeapWord*) obj)), "invariant"); scan_object(obj); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/concurrentMark.hpp --- a/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,7 +25,7 @@ #ifndef SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_HPP #define SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_HPP -#include "gc_implementation/g1/heapRegion.hpp" +#include "gc_implementation/g1/heapRegionSets.hpp" #include "utilities/taskqueue.hpp" class G1CollectedHeap; @@ -369,13 +369,7 @@ double _cleanup_sleep_factor; double _cleanup_task_overhead; - // Stuff related to age cohort processing. - struct ParCleanupThreadState { - char _pre[64]; - UncleanRegionList list; - char _post[64]; - }; - ParCleanupThreadState** _par_cleanup_thread_state; + FreeRegionList _cleanup_list; // CMS marking support structures CMBitMap _markBitMap1; @@ -484,6 +478,10 @@ // prints all gathered CM-related statistics void print_stats(); + bool cleanup_list_is_empty() { + return _cleanup_list.is_empty(); + } + // accessor methods size_t parallel_marking_threads() { return _parallel_marking_threads; } double sleep_factor() { return _sleep_factor; } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp --- a/hotspot/src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -95,8 +95,8 @@ _vtime_start = os::elapsedVTime(); wait_for_universe_init(); - G1CollectedHeap* g1 = G1CollectedHeap::heap(); - G1CollectorPolicy* g1_policy = g1->g1_policy(); + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + G1CollectorPolicy* g1_policy = g1h->g1_policy(); G1MMUTracker *mmu_tracker = g1_policy->mmu_tracker(); Thread *current_thread = Thread::current(); @@ -119,7 +119,7 @@ if (!g1_policy->in_young_gc_mode()) { // this ensures the flag is not set if we bail out of the marking // cycle; normally the flag is cleared immediately after cleanup - g1->set_marking_complete(); + g1h->set_marking_complete(); if (g1_policy->adaptive_young_list_length()) { double now = os::elapsedTime(); @@ -228,10 +228,20 @@ VM_CGC_Operation op(&cl_cl, verbose_str); VMThread::execute(&op); } else { - G1CollectedHeap::heap()->set_marking_complete(); + g1h->set_marking_complete(); } - if (!cm()->has_aborted()) { + // Check if cleanup set the free_regions_coming flag. If it + // hasn't, we can just skip the next step. + if (g1h->free_regions_coming()) { + // The following will finish freeing up any regions that we + // found to be empty during cleanup. We'll do this part + // without joining the suspendible set. If an evacuation pause + // takes places, then we would carry on freeing regions in + // case they are needed by the pause. If a Full GC takes + // places, it would wait for us to process the regions + // reclaimed by cleanup. + double cleanup_start_sec = os::elapsedTime(); if (PrintGC) { gclog_or_tty->date_stamp(PrintGCDateStamps); @@ -240,23 +250,22 @@ } // Now do the remainder of the cleanup operation. - _sts.join(); _cm->completeCleanup(); - if (!cm()->has_aborted()) { - g1_policy->record_concurrent_mark_cleanup_completed(); + g1_policy->record_concurrent_mark_cleanup_completed(); - double cleanup_end_sec = os::elapsedTime(); - if (PrintGC) { - gclog_or_tty->date_stamp(PrintGCDateStamps); - gclog_or_tty->stamp(PrintGCTimeStamps); - gclog_or_tty->print_cr("[GC concurrent-cleanup-end, %1.7lf]", - cleanup_end_sec - cleanup_start_sec); - } + double cleanup_end_sec = os::elapsedTime(); + if (PrintGC) { + gclog_or_tty->date_stamp(PrintGCDateStamps); + gclog_or_tty->stamp(PrintGCTimeStamps); + gclog_or_tty->print_cr("[GC concurrent-cleanup-end, %1.7lf]", + cleanup_end_sec - cleanup_start_sec); } - _sts.leave(); + + // We're done: no more free regions coming. + g1h->reset_free_regions_coming(); } - // We're done: no more unclean regions coming. - G1CollectedHeap::heap()->set_unclean_regions_coming(false); + guarantee(cm()->cleanup_list_is_empty(), + "at this point there should be no regions on the cleanup list"); if (cm()->has_aborted()) { if (PrintGC) { @@ -278,7 +287,7 @@ // Java thread is waiting for a full GC to happen (e.g., it // called System.gc() with +ExplicitGCInvokesConcurrent). _sts.join(); - g1->increment_full_collections_completed(true /* concurrent */); + g1h->increment_full_collections_completed(true /* concurrent */); _sts.leave(); } assert(_should_terminate, "just checking"); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/concurrentZFThread.cpp --- a/hotspot/src/share/vm/gc_implementation/g1/concurrentZFThread.cpp Thu Jan 27 17:28:34 2011 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,194 +0,0 @@ -/* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/concurrentZFThread.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/heapRegion.hpp" -#include "memory/space.inline.hpp" -#include "runtime/mutexLocker.hpp" -#include "utilities/copy.hpp" - -// ======= Concurrent Zero-Fill Thread ======== - -// The CM thread is created when the G1 garbage collector is used - -int ConcurrentZFThread::_region_allocs = 0; -int ConcurrentZFThread::_sync_zfs = 0; -int ConcurrentZFThread::_zf_waits = 0; -int ConcurrentZFThread::_regions_filled = 0; - -ConcurrentZFThread::ConcurrentZFThread() : - ConcurrentGCThread() -{ - create_and_start(); -} - -void ConcurrentZFThread::wait_for_ZF_completed(HeapRegion* hr) { - assert(ZF_mon->owned_by_self(), "Precondition."); - note_zf_wait(); - while (hr->zero_fill_state() == HeapRegion::ZeroFilling) { - ZF_mon->wait(Mutex::_no_safepoint_check_flag); - } -} - -void ConcurrentZFThread::processHeapRegion(HeapRegion* hr) { - assert(!Universe::heap()->is_gc_active(), - "This should not happen during GC."); - assert(hr != NULL, "Precondition"); - // These are unlocked reads, but if this test is successful, then no - // other thread will attempt this zero filling. Only a GC thread can - // modify the ZF state of a region whose state is zero-filling, and this - // should only happen while the ZF thread is locking out GC. - if (hr->zero_fill_state() == HeapRegion::ZeroFilling - && hr->zero_filler() == Thread::current()) { - assert(hr->top() == hr->bottom(), "better be empty!"); - assert(!hr->isHumongous(), "Only free regions on unclean list."); - Copy::fill_to_words(hr->bottom(), hr->capacity()/HeapWordSize); - note_region_filled(); - } -} - -void ConcurrentZFThread::run() { - initialize_in_thread(); - Thread* thr_self = Thread::current(); - _vtime_start = os::elapsedVTime(); - wait_for_universe_init(); - - G1CollectedHeap* g1 = G1CollectedHeap::heap(); - _sts.join(); - while (!_should_terminate) { - _sts.leave(); - - { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - - // This local variable will hold a region being zero-filled. This - // region will neither be on the unclean or zero-filled lists, and - // will not be available for allocation; thus, we might have an - // allocation fail, causing a full GC, because of this, but this is a - // price we will pay. (In future, we might want to make the fact - // that there's a region being zero-filled apparent to the G1 heap, - // which could then wait for it in this extreme case...) - HeapRegion* to_fill; - - while (!g1->should_zf() - || (to_fill = g1->pop_unclean_region_list_locked()) == NULL) - ZF_mon->wait(Mutex::_no_safepoint_check_flag); - while (to_fill->zero_fill_state() == HeapRegion::ZeroFilling) - ZF_mon->wait(Mutex::_no_safepoint_check_flag); - - // So now to_fill is non-NULL and is not ZeroFilling. It might be - // Allocated or ZeroFilled. (The latter could happen if this thread - // starts the zero-filling of a region, but a GC intervenes and - // pushes new regions needing on the front of the filling on the - // front of the list.) - - switch (to_fill->zero_fill_state()) { - case HeapRegion::Allocated: - to_fill = NULL; - break; - - case HeapRegion::NotZeroFilled: - to_fill->set_zero_fill_in_progress(thr_self); - - ZF_mon->unlock(); - _sts.join(); - processHeapRegion(to_fill); - _sts.leave(); - ZF_mon->lock_without_safepoint_check(); - - if (to_fill->zero_fill_state() == HeapRegion::ZeroFilling - && to_fill->zero_filler() == thr_self) { - to_fill->set_zero_fill_complete(); - (void)g1->put_free_region_on_list_locked(to_fill); - } - break; - - case HeapRegion::ZeroFilled: - (void)g1->put_free_region_on_list_locked(to_fill); - break; - - case HeapRegion::ZeroFilling: - ShouldNotReachHere(); - break; - } - } - _vtime_accum = (os::elapsedVTime() - _vtime_start); - _sts.join(); - } - _sts.leave(); - - assert(_should_terminate, "just checking"); - terminate(); -} - -bool ConcurrentZFThread::offer_yield() { - if (_sts.should_yield()) { - _sts.yield("Concurrent ZF"); - return true; - } else { - return false; - } -} - -void ConcurrentZFThread::stop() { - // it is ok to take late safepoints here, if needed - MutexLockerEx mu(Terminator_lock); - _should_terminate = true; - while (!_has_terminated) { - Terminator_lock->wait(); - } -} - -void ConcurrentZFThread::print() const { - print_on(tty); -} - -void ConcurrentZFThread::print_on(outputStream* st) const { - st->print("\"G1 Concurrent Zero-Fill Thread\" "); - Thread::print_on(st); - st->cr(); -} - - -double ConcurrentZFThread::_vtime_accum; - -void ConcurrentZFThread::print_summary_info() { - gclog_or_tty->print("\nConcurrent Zero-Filling:\n"); - gclog_or_tty->print(" Filled %d regions, used %5.2fs.\n", - _regions_filled, - vtime_accum()); - gclog_or_tty->print(" Of %d region allocs, %d (%5.2f%%) required sync ZF,\n", - _region_allocs, _sync_zfs, - (_region_allocs > 0 ? - (float)_sync_zfs/(float)_region_allocs*100.0 : - 0.0)); - gclog_or_tty->print(" and %d (%5.2f%%) required a ZF wait.\n", - _zf_waits, - (_region_allocs > 0 ? - (float)_zf_waits/(float)_region_allocs*100.0 : - 0.0)); - -} diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/concurrentZFThread.hpp --- a/hotspot/src/share/vm/gc_implementation/g1/concurrentZFThread.hpp Thu Jan 27 17:28:34 2011 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,91 +0,0 @@ -/* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTZFTHREAD_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTZFTHREAD_HPP - -#include "gc_implementation/shared/concurrentGCThread.hpp" - -// The Concurrent ZF Thread. Performs concurrent zero-filling. - -class ConcurrentZFThread: public ConcurrentGCThread { - friend class VMStructs; - friend class ZeroFillRegionClosure; - - private: - - // Zero fill the heap region. - void processHeapRegion(HeapRegion* r); - - // Stats - // Allocation (protected by heap lock). - static int _region_allocs; // Number of regions allocated - static int _sync_zfs; // Synchronous zero-fills + - static int _zf_waits; // Wait for conc zero-fill completion. - - // Number of regions CFZ thread fills. - static int _regions_filled; - - double _vtime_start; // Initial virtual time. - - // These are static because the "print_summary_info" method is, and - // it currently assumes there is only one ZF thread. We'll change when - // we need to. - static double _vtime_accum; // Initial virtual time. - static double vtime_accum() { return _vtime_accum; } - - // Offer yield for GC. Returns true if yield occurred. - bool offer_yield(); - - public: - // Constructor - ConcurrentZFThread(); - - // Main loop. - virtual void run(); - - // Printing - void print_on(outputStream* st) const; - void print() const; - - // Waits until "r" has been zero-filled. Requires caller to hold the - // ZF_mon. - static void wait_for_ZF_completed(HeapRegion* r); - - // Get or clear the current unclean region. Should be done - // while holding the ZF_needed_mon lock. - - // shutdown - void stop(); - - // Stats - static void note_region_alloc() {_region_allocs++; } - static void note_sync_zfs() { _sync_zfs++; } - static void note_zf_wait() { _zf_waits++; } - static void note_region_filled() { _regions_filled++; } - - static void print_summary_info(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTZFTHREAD_HPP diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp --- a/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -222,7 +222,7 @@ // Action_mark - update the BOT for the block [blk_start, blk_end). // Current typical use is for splitting a block. -// Action_single - udpate the BOT for an allocation. +// Action_single - update the BOT for an allocation. // Action_verify - BOT verification. void G1BlockOffsetArray::do_block_internal(HeapWord* blk_start, HeapWord* blk_end, @@ -331,47 +331,6 @@ do_block_internal(blk_start, blk_end, Action_mark); } -void G1BlockOffsetArray::join_blocks(HeapWord* blk1, HeapWord* blk2) { - HeapWord* blk1_start = Universe::heap()->block_start(blk1); - HeapWord* blk2_start = Universe::heap()->block_start(blk2); - assert(blk1 == blk1_start && blk2 == blk2_start, - "Must be block starts."); - assert(blk1 + _sp->block_size(blk1) == blk2, "Must be contiguous."); - size_t blk1_start_index = _array->index_for(blk1); - size_t blk2_start_index = _array->index_for(blk2); - assert(blk1_start_index <= blk2_start_index, "sanity"); - HeapWord* blk2_card_start = _array->address_for_index(blk2_start_index); - if (blk2 == blk2_card_start) { - // blk2 starts a card. Does blk1 start on the prevous card, or futher - // back? - assert(blk1_start_index < blk2_start_index, "must be lower card."); - if (blk1_start_index + 1 == blk2_start_index) { - // previous card; new value for blk2 card is size of blk1. - _array->set_offset_array(blk2_start_index, (u_char) _sp->block_size(blk1)); - } else { - // Earlier card; go back a card. - _array->set_offset_array(blk2_start_index, N_words); - } - } else { - // blk2 does not start a card. Does it cross a card? If not, nothing - // to do. - size_t blk2_end_index = - _array->index_for(blk2 + _sp->block_size(blk2) - 1); - assert(blk2_end_index >= blk2_start_index, "sanity"); - if (blk2_end_index > blk2_start_index) { - // Yes, it crosses a card. The value for the next card must change. - if (blk1_start_index + 1 == blk2_start_index) { - // previous card; new value for second blk2 card is size of blk1. - _array->set_offset_array(blk2_start_index + 1, - (u_char) _sp->block_size(blk1)); - } else { - // Earlier card; go back a card. - _array->set_offset_array(blk2_start_index + 1, N_words); - } - } - } -} - HeapWord* G1BlockOffsetArray::block_start_unsafe(const void* addr) { assert(_bottom <= addr && addr < _end, "addr must be covered by this Array"); @@ -580,15 +539,50 @@ #endif } -void -G1BlockOffsetArray::set_for_starts_humongous(HeapWord* new_end) { - assert(_end == new_end, "_end should have already been updated"); +bool +G1BlockOffsetArray::verify_for_object(HeapWord* obj_start, + size_t word_size) const { + size_t first_card = _array->index_for(obj_start); + size_t last_card = _array->index_for(obj_start + word_size - 1); + if (!_array->is_card_boundary(obj_start)) { + // If the object is not on a card boundary the BOT entry of the + // first card should point to another object so we should not + // check that one. + first_card += 1; + } + for (size_t card = first_card; card <= last_card; card += 1) { + HeapWord* card_addr = _array->address_for_index(card); + HeapWord* block_start = block_start_const(card_addr); + if (block_start != obj_start) { + gclog_or_tty->print_cr("block start: "PTR_FORMAT" is incorrect - " + "card index: "SIZE_FORMAT" " + "card addr: "PTR_FORMAT" BOT entry: %u " + "obj: "PTR_FORMAT" word size: "SIZE_FORMAT" " + "cards: ["SIZE_FORMAT","SIZE_FORMAT"]", + block_start, card, card_addr, + _array->offset_array(card), + obj_start, word_size, first_card, last_card); + return false; + } + } + return true; +} - // The first BOT entry should have offset 0. - _array->set_offset_array(_array->index_for(_bottom), 0); - // The rest should point to the first one. - set_remainder_to_point_to_start(_bottom + N_words, new_end); +#ifndef PRODUCT +void +G1BlockOffsetArray::print_on(outputStream* out) { + size_t from_index = _array->index_for(_bottom); + size_t to_index = _array->index_for(_end); + out->print_cr(">> BOT for area ["PTR_FORMAT","PTR_FORMAT") " + "cards ["SIZE_FORMAT","SIZE_FORMAT")", + _bottom, _end, from_index, to_index); + for (size_t i = from_index; i < to_index; ++i) { + out->print_cr(" entry "SIZE_FORMAT_W(8)" | "PTR_FORMAT" : %3u", + i, _array->address_for_index(i), + (uint) _array->offset_array(i)); + } } +#endif // !PRODUCT ////////////////////////////////////////////////////////////////////// // G1BlockOffsetArrayContigSpace @@ -641,10 +635,20 @@ } void -G1BlockOffsetArrayContigSpace::set_for_starts_humongous(HeapWord* new_end) { - G1BlockOffsetArray::set_for_starts_humongous(new_end); +G1BlockOffsetArrayContigSpace::set_for_starts_humongous(HeapWord* new_top) { + assert(new_top <= _end, "_end should have already been updated"); + + // The first BOT entry should have offset 0. + zero_bottom_entry(); + initialize_threshold(); + alloc_block(_bottom, new_top); + } - // Make sure _next_offset_threshold and _next_offset_index point to new_end. - _next_offset_threshold = new_end; - _next_offset_index = _array->index_for(new_end); +#ifndef PRODUCT +void +G1BlockOffsetArrayContigSpace::print_on(outputStream* out) { + G1BlockOffsetArray::print_on(out); + out->print_cr(" next offset threshold: "PTR_FORMAT, _next_offset_threshold); + out->print_cr(" next offset index: "SIZE_FORMAT, _next_offset_index); } +#endif // !PRODUCT diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.hpp --- a/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -352,11 +352,6 @@ // The following methods are useful and optimized for a // general, non-contiguous space. - // The given arguments are required to be the starts of adjacent ("blk1" - // before "blk2") well-formed blocks covered by "this". After this call, - // they should be considered to form one block. - virtual void join_blocks(HeapWord* blk1, HeapWord* blk2); - // Given a block [blk_start, blk_start + full_blk_size), and // a left_blk_size < full_blk_size, adjust the BOT to show two // blocks [blk_start, blk_start + left_blk_size) and @@ -429,6 +424,12 @@ verify_single_block(blk, blk + size); } + // Used by region verification. Checks that the contents of the + // BOT reflect that there's a single object that spans the address + // range [obj_start, obj_start + word_size); returns true if this is + // the case, returns false if it's not. + bool verify_for_object(HeapWord* obj_start, size_t word_size) const; + // Verify that the given block is before _unallocated_block inline void verify_not_unallocated(HeapWord* blk_start, HeapWord* blk_end) const { @@ -444,7 +445,7 @@ void check_all_cards(size_t left_card, size_t right_card) const; - virtual void set_for_starts_humongous(HeapWord* new_end); + virtual void print_on(outputStream* out) PRODUCT_RETURN; }; // A subtype of BlockOffsetArray that takes advantage of the fact @@ -494,7 +495,9 @@ HeapWord* block_start_unsafe(const void* addr); HeapWord* block_start_unsafe_const(const void* addr) const; - virtual void set_for_starts_humongous(HeapWord* new_end); + void set_for_starts_humongous(HeapWord* new_top); + + virtual void print_on(outputStream* out) PRODUCT_RETURN; }; #endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1BLOCKOFFSETTABLE_HPP diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp --- a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,7 +28,6 @@ #include "gc_implementation/g1/concurrentG1Refine.hpp" #include "gc_implementation/g1/concurrentG1RefineThread.hpp" #include "gc_implementation/g1/concurrentMarkThread.inline.hpp" -#include "gc_implementation/g1/concurrentZFThread.hpp" #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" #include "gc_implementation/g1/g1CollectorPolicy.hpp" #include "gc_implementation/g1/g1MarkSweep.hpp" @@ -425,11 +424,9 @@ void G1CollectedHeap::stop_conc_gc_threads() { _cg1r->stop(); - _czft->stop(); _cmThread->stop(); } - void G1CollectedHeap::check_ct_logs_at_safepoint() { DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); CardTableModRefBS* ct_bs = (CardTableModRefBS*)barrier_set(); @@ -481,49 +478,92 @@ // Private methods. -// Finds a HeapRegion that can be used to allocate a given size of block. - - -HeapRegion* G1CollectedHeap::newAllocRegion_work(size_t word_size, - bool do_expand, - bool zero_filled) { - ConcurrentZFThread::note_region_alloc(); - HeapRegion* res = alloc_free_region_from_lists(zero_filled); +HeapRegion* +G1CollectedHeap::new_region_try_secondary_free_list(size_t word_size) { + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + while (!_secondary_free_list.is_empty() || free_regions_coming()) { + if (!_secondary_free_list.is_empty()) { + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " + "secondary_free_list has "SIZE_FORMAT" entries", + _secondary_free_list.length()); + } + // It looks as if there are free regions available on the + // secondary_free_list. Let's move them to the free_list and try + // again to allocate from it. + append_secondary_free_list(); + + assert(!_free_list.is_empty(), "if the secondary_free_list was not " + "empty we should have moved at least one entry to the free_list"); + HeapRegion* res = _free_list.remove_head(); + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " + "allocated "HR_FORMAT" from secondary_free_list", + HR_FORMAT_PARAMS(res)); + } + return res; + } + + // Wait here until we get notifed either when (a) there are no + // more free regions coming or (b) some regions have been moved on + // the secondary_free_list. + SecondaryFreeList_lock->wait(Mutex::_no_safepoint_check_flag); + } + + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " + "could not allocate from secondary_free_list"); + } + return NULL; +} + +HeapRegion* G1CollectedHeap::new_region_work(size_t word_size, + bool do_expand) { + assert(!isHumongous(word_size) || + word_size <= (size_t) HeapRegion::GrainWords, + "the only time we use this to allocate a humongous region is " + "when we are allocating a single humongous region"); + + HeapRegion* res; + if (G1StressConcRegionFreeing) { + if (!_secondary_free_list.is_empty()) { + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " + "forced to look at the secondary_free_list"); + } + res = new_region_try_secondary_free_list(word_size); + if (res != NULL) { + return res; + } + } + } + res = _free_list.remove_head_or_null(); + if (res == NULL) { + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " + "res == NULL, trying the secondary_free_list"); + } + res = new_region_try_secondary_free_list(word_size); + } if (res == NULL && do_expand) { expand(word_size * HeapWordSize); - res = alloc_free_region_from_lists(zero_filled); - assert(res == NULL || - (!res->isHumongous() && - (!zero_filled || - res->zero_fill_state() == HeapRegion::Allocated)), - "Alloc Regions must be zero filled (and non-H)"); + res = _free_list.remove_head_or_null(); } if (res != NULL) { - if (res->is_empty()) { - _free_regions--; - } - assert(!res->isHumongous() && - (!zero_filled || res->zero_fill_state() == HeapRegion::Allocated), - err_msg("Non-young alloc Regions must be zero filled (and non-H):" - " res->isHumongous()=%d, zero_filled=%d, res->zero_fill_state()=%d", - res->isHumongous(), zero_filled, res->zero_fill_state())); - assert(!res->is_on_unclean_list(), - "Alloc Regions must not be on the unclean list"); if (G1PrintHeapRegions) { - gclog_or_tty->print_cr("new alloc region %d:["PTR_FORMAT", "PTR_FORMAT"], " - "top "PTR_FORMAT, - res->hrs_index(), res->bottom(), res->end(), res->top()); + gclog_or_tty->print_cr("new alloc region %d:["PTR_FORMAT","PTR_FORMAT"], " + "top "PTR_FORMAT, res->hrs_index(), + res->bottom(), res->end(), res->top()); } } return res; } -HeapRegion* G1CollectedHeap::newAllocRegionWithExpansion(int purpose, - size_t word_size, - bool zero_filled) { +HeapRegion* G1CollectedHeap::new_gc_alloc_region(int purpose, + size_t word_size) { HeapRegion* alloc_region = NULL; if (_gc_alloc_region_counts[purpose] < g1_policy()->max_regions(purpose)) { - alloc_region = newAllocRegion_work(word_size, true, zero_filled); + alloc_region = new_region_work(word_size, true /* do_expand */); if (purpose == GCAllocForSurvived && alloc_region != NULL) { alloc_region->set_survivor(); } @@ -534,81 +574,220 @@ return alloc_region; } +int G1CollectedHeap::humongous_obj_allocate_find_first(size_t num_regions, + size_t word_size) { + int first = -1; + if (num_regions == 1) { + // Only one region to allocate, no need to go through the slower + // path. The caller will attempt the expasion if this fails, so + // let's not try to expand here too. + HeapRegion* hr = new_region_work(word_size, false /* do_expand */); + if (hr != NULL) { + first = hr->hrs_index(); + } else { + first = -1; + } + } else { + // We can't allocate humongous regions while cleanupComplete() is + // running, since some of the regions we find to be empty might not + // yet be added to the free list and it is not straightforward to + // know which list they are on so that we can remove them. Note + // that we only need to do this if we need to allocate more than + // one region to satisfy the current humongous allocation + // request. If we are only allocating one region we use the common + // region allocation code (see above). + wait_while_free_regions_coming(); + append_secondary_free_list_if_not_empty(); + + if (free_regions() >= num_regions) { + first = _hrs->find_contiguous(num_regions); + if (first != -1) { + for (int i = first; i < first + (int) num_regions; ++i) { + HeapRegion* hr = _hrs->at(i); + assert(hr->is_empty(), "sanity"); + assert(is_on_free_list(hr), "sanity"); + hr->set_pending_removal(true); + } + _free_list.remove_all_pending(num_regions); + } + } + } + return first; +} + // If could fit into free regions w/o expansion, try. // Otherwise, if can expand, do so. // Otherwise, if using ex regions might help, try with ex given back. HeapWord* G1CollectedHeap::humongous_obj_allocate(size_t word_size) { - assert_heap_locked_or_at_safepoint(); - assert(regions_accounted_for(), "Region leakage!"); - - // We can't allocate humongous regions while cleanupComplete is - // running, since some of the regions we find to be empty might not - // yet be added to the unclean list. If we're already at a - // safepoint, this call is unnecessary, not to mention wrong. - if (!SafepointSynchronize::is_at_safepoint()) { - wait_for_cleanup_complete(); - } + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); + + verify_region_sets_optional(); size_t num_regions = round_to(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords; - - // Special case if < one region??? - - // Remember the ft size. size_t x_size = expansion_regions(); - - HeapWord* res = NULL; - bool eliminated_allocated_from_lists = false; - - // Can the allocation potentially fit in the free regions? - if (free_regions() >= num_regions) { - res = _hrs->obj_allocate(word_size); - } - if (res == NULL) { - // Try expansion. - size_t fs = _hrs->free_suffix(); + size_t fs = _hrs->free_suffix(); + int first = humongous_obj_allocate_find_first(num_regions, word_size); + if (first == -1) { + // The only thing we can do now is attempt expansion. if (fs + x_size >= num_regions) { expand((num_regions - fs) * HeapRegion::GrainBytes); - res = _hrs->obj_allocate(word_size); - assert(res != NULL, "This should have worked."); - } else { - // Expansion won't help. Are there enough free regions if we get rid - // of reservations? - size_t avail = free_regions(); - if (avail >= num_regions) { - res = _hrs->obj_allocate(word_size); - if (res != NULL) { - remove_allocated_regions_from_lists(); - eliminated_allocated_from_lists = true; - } + first = humongous_obj_allocate_find_first(num_regions, word_size); + assert(first != -1, "this should have worked"); + } + } + + if (first != -1) { + // Index of last region in the series + 1. + int last = first + (int) num_regions; + + // We need to initialize the region(s) we just discovered. This is + // a bit tricky given that it can happen concurrently with + // refinement threads refining cards on these regions and + // potentially wanting to refine the BOT as they are scanning + // those cards (this can happen shortly after a cleanup; see CR + // 6991377). So we have to set up the region(s) carefully and in + // a specific order. + + // The word size sum of all the regions we will allocate. + size_t word_size_sum = num_regions * HeapRegion::GrainWords; + assert(word_size <= word_size_sum, "sanity"); + + // This will be the "starts humongous" region. + HeapRegion* first_hr = _hrs->at(first); + // The header of the new object will be placed at the bottom of + // the first region. + HeapWord* new_obj = first_hr->bottom(); + // This will be the new end of the first region in the series that + // should also match the end of the last region in the seriers. + HeapWord* new_end = new_obj + word_size_sum; + // This will be the new top of the first region that will reflect + // this allocation. + HeapWord* new_top = new_obj + word_size; + + // First, we need to zero the header of the space that we will be + // allocating. When we update top further down, some refinement + // threads might try to scan the region. By zeroing the header we + // ensure that any thread that will try to scan the region will + // come across the zero klass word and bail out. + // + // NOTE: It would not have been correct to have used + // CollectedHeap::fill_with_object() and make the space look like + // an int array. The thread that is doing the allocation will + // later update the object header to a potentially different array + // type and, for a very short period of time, the klass and length + // fields will be inconsistent. This could cause a refinement + // thread to calculate the object size incorrectly. + Copy::fill_to_words(new_obj, oopDesc::header_size(), 0); + + // We will set up the first region as "starts humongous". This + // will also update the BOT covering all the regions to reflect + // that there is a single object that starts at the bottom of the + // first region. + first_hr->set_startsHumongous(new_top, new_end); + + // Then, if there are any, we will set up the "continues + // humongous" regions. + HeapRegion* hr = NULL; + for (int i = first + 1; i < last; ++i) { + hr = _hrs->at(i); + hr->set_continuesHumongous(first_hr); + } + // If we have "continues humongous" regions (hr != NULL), then the + // end of the last one should match new_end. + assert(hr == NULL || hr->end() == new_end, "sanity"); + + // Up to this point no concurrent thread would have been able to + // do any scanning on any region in this series. All the top + // fields still point to bottom, so the intersection between + // [bottom,top] and [card_start,card_end] will be empty. Before we + // update the top fields, we'll do a storestore to make sure that + // no thread sees the update to top before the zeroing of the + // object header and the BOT initialization. + OrderAccess::storestore(); + + // Now that the BOT and the object header have been initialized, + // we can update top of the "starts humongous" region. + assert(first_hr->bottom() < new_top && new_top <= first_hr->end(), + "new_top should be in this region"); + first_hr->set_top(new_top); + + // Now, we will update the top fields of the "continues humongous" + // regions. The reason we need to do this is that, otherwise, + // these regions would look empty and this will confuse parts of + // G1. For example, the code that looks for a consecutive number + // of empty regions will consider them empty and try to + // re-allocate them. We can extend is_empty() to also include + // !continuesHumongous(), but it is easier to just update the top + // fields here. The way we set top for all regions (i.e., top == + // end for all regions but the last one, top == new_top for the + // last one) is actually used when we will free up the humongous + // region in free_humongous_region(). + hr = NULL; + for (int i = first + 1; i < last; ++i) { + hr = _hrs->at(i); + if ((i + 1) == last) { + // last continues humongous region + assert(hr->bottom() < new_top && new_top <= hr->end(), + "new_top should fall on this region"); + hr->set_top(new_top); + } else { + // not last one + assert(new_top > hr->end(), "new_top should be above this region"); + hr->set_top(hr->end()); } } - } - if (res != NULL) { - // Increment by the number of regions allocated. - // FIXME: Assumes regions all of size GrainBytes. -#ifndef PRODUCT - mr_bs()->verify_clean_region(MemRegion(res, res + num_regions * - HeapRegion::GrainWords)); -#endif - if (!eliminated_allocated_from_lists) - remove_allocated_regions_from_lists(); - _summary_bytes_used += word_size * HeapWordSize; - _free_regions -= num_regions; - _num_humongous_regions += (int) num_regions; - } - assert(regions_accounted_for(), "Region Leakage"); - return res; + // If we have continues humongous regions (hr != NULL), then the + // end of the last one should match new_end and its top should + // match new_top. + assert(hr == NULL || + (hr->end() == new_end && hr->top() == new_top), "sanity"); + + assert(first_hr->used() == word_size * HeapWordSize, "invariant"); + _summary_bytes_used += first_hr->used(); + _humongous_set.add(first_hr); + + return new_obj; + } + + verify_region_sets_optional(); + return NULL; } void G1CollectedHeap::retire_cur_alloc_region(HeapRegion* cur_alloc_region) { - // The cleanup operation might update _summary_bytes_used - // concurrently with this method. So, right now, if we don't wait - // for it to complete, updates to _summary_bytes_used might get - // lost. This will be resolved in the near future when the operation - // of the free region list is revamped as part of CR 6977804. - wait_for_cleanup_complete(); + // Other threads might still be trying to allocate using CASes out + // of the region we are retiring, as they can do so without holding + // the Heap_lock. So we first have to make sure that noone else can + // allocate in it by doing a maximal allocation. Even if our CAS + // attempt fails a few times, we'll succeed sooner or later given + // that a failed CAS attempt mean that the region is getting closed + // to being full (someone else succeeded in allocating into it). + size_t free_word_size = cur_alloc_region->free() / HeapWordSize; + + // This is the minimum free chunk we can turn into a dummy + // object. If the free space falls below this, then noone can + // allocate in this region anyway (all allocation requests will be + // of a size larger than this) so we won't have to perform the dummy + // allocation. + size_t min_word_size_to_fill = CollectedHeap::min_fill_size(); + + while (free_word_size >= min_word_size_to_fill) { + HeapWord* dummy = + cur_alloc_region->par_allocate_no_bot_updates(free_word_size); + if (dummy != NULL) { + // If the allocation was successful we should fill in the space. + CollectedHeap::fill_with_object(dummy, free_word_size); + break; + } + + free_word_size = cur_alloc_region->free() / HeapWordSize; + // It's also possible that someone else beats us to the + // allocation and they fill up the region. In that case, we can + // just get out of the loop + } + assert(cur_alloc_region->free() / HeapWordSize < min_word_size_to_fill, + "sanity"); retire_cur_alloc_region_common(cur_alloc_region); assert(_cur_alloc_region == NULL, "post-condition"); @@ -621,7 +800,7 @@ bool at_safepoint, bool do_dirtying, bool can_expand) { - assert_heap_locked_or_at_safepoint(); + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); assert(_cur_alloc_region == NULL, "replace_cur_alloc_region_and_allocate() should only be called " "after retiring the previous current alloc region"); @@ -632,25 +811,12 @@ "we are not allowed to expand the young gen"); if (can_expand || !g1_policy()->is_young_list_full()) { - if (!at_safepoint) { - // The cleanup operation might update _summary_bytes_used - // concurrently with this method. So, right now, if we don't - // wait for it to complete, updates to _summary_bytes_used might - // get lost. This will be resolved in the near future when the - // operation of the free region list is revamped as part of - // CR 6977804. If we're already at a safepoint, this call is - // unnecessary, not to mention wrong. - wait_for_cleanup_complete(); - } - - HeapRegion* new_cur_alloc_region = newAllocRegion(word_size, - false /* zero_filled */); + HeapRegion* new_cur_alloc_region = new_alloc_region(word_size); if (new_cur_alloc_region != NULL) { assert(new_cur_alloc_region->is_empty(), "the newly-allocated region should be empty, " "as right now we only allocate new regions out of the free list"); g1_policy()->update_region_num(true /* next_is_young */); - _summary_bytes_used -= new_cur_alloc_region->used(); set_region_short_lived_locked(new_cur_alloc_region); assert(!new_cur_alloc_region->isHumongous(), @@ -661,27 +827,29 @@ // young type. OrderAccess::storestore(); - // Now allocate out of the new current alloc region. We could - // have re-used allocate_from_cur_alloc_region() but its - // operation is slightly different to what we need here. First, - // allocate_from_cur_alloc_region() is only called outside a - // safepoint and will always unlock the Heap_lock if it returns - // a non-NULL result. Second, it assumes that the current alloc - // region is what's already assigned in _cur_alloc_region. What - // we want here is to actually do the allocation first before we - // assign the new region to _cur_alloc_region. This ordering is - // not currently important, but it will be essential when we - // change the code to support CAS allocation in the future (see - // CR 6994297). - // - // This allocate method does BOT updates and we don't need them in - // the young generation. This will be fixed in the near future by - // CR 6994297. - HeapWord* result = new_cur_alloc_region->allocate(word_size); + // Now, perform the allocation out of the region we just + // allocated. Note that noone else can access that region at + // this point (as _cur_alloc_region has not been updated yet), + // so we can just go ahead and do the allocation without any + // atomics (and we expect this allocation attempt to + // suceeded). Given that other threads can attempt an allocation + // with a CAS and without needing the Heap_lock, if we assigned + // the new region to _cur_alloc_region before first allocating + // into it other threads might have filled up the new region + // before we got a chance to do the allocation ourselves. In + // that case, we would have needed to retire the region, grab a + // new one, and go through all this again. Allocating out of the + // new region before assigning it to _cur_alloc_region avoids + // all this. + HeapWord* result = + new_cur_alloc_region->allocate_no_bot_updates(word_size); assert(result != NULL, "we just allocate out of an empty region " "so allocation should have been successful"); assert(is_in(result), "result should be in the heap"); + // Now make sure that the store to _cur_alloc_region does not + // float above the store to top. + OrderAccess::storestore(); _cur_alloc_region = new_cur_alloc_region; if (!at_safepoint) { @@ -698,7 +866,7 @@ assert(_cur_alloc_region == NULL, "we failed to allocate a new current " "alloc region, it should still be NULL"); - assert_heap_locked_or_at_safepoint(); + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); return NULL; } @@ -710,6 +878,10 @@ assert(!isHumongous(word_size), "attempt_allocation_slow() should not be " "used for humongous allocations"); + // We should only reach here when we were unable to allocate + // otherwise. So, we should have not active current alloc region. + assert(_cur_alloc_region == NULL, "current alloc region should be NULL"); + // We will loop while succeeded is false, which means that we tried // to do a collection, but the VM op did not succeed. So, when we // exit the loop, either one of the allocation attempts was @@ -718,28 +890,8 @@ for (int try_count = 1; /* we'll return or break */; try_count += 1) { bool succeeded = true; - { - // We may have concurrent cleanup working at the time. Wait for - // it to complete. In the future we would probably want to make - // the concurrent cleanup truly concurrent by decoupling it from - // the allocation. This will happen in the near future as part - // of CR 6977804 which will revamp the operation of the free - // region list. The fact that wait_for_cleanup_complete() will - // do a wait() means that we'll give up the Heap_lock. So, it's - // possible that when we exit wait_for_cleanup_complete() we - // might be able to allocate successfully (since somebody else - // might have done a collection meanwhile). So, we'll attempt to - // allocate again, just in case. When we make cleanup truly - // concurrent with allocation, we should remove this allocation - // attempt as it's redundant (we only reach here after an - // allocation attempt has been unsuccessful). - wait_for_cleanup_complete(); - HeapWord* result = attempt_allocation(word_size); - if (result != NULL) { - assert_heap_not_locked(); - return result; - } - } + // Every time we go round the loop we should be holding the Heap_lock. + assert_heap_locked(); if (GC_locker::is_active_and_needs_gc()) { // We are locked out of GC because of the GC locker. We can @@ -748,7 +900,6 @@ if (g1_policy()->can_expand_young_list()) { // Yes, we are allowed to expand the young gen. Let's try to // allocate a new current alloc region. - HeapWord* result = replace_cur_alloc_region_and_allocate(word_size, false, /* at_safepoint */ @@ -771,20 +922,23 @@ // rather than causing more, now probably unnecessary, GC attempts. JavaThread* jthr = JavaThread::current(); assert(jthr != NULL, "sanity"); - if (!jthr->in_critical()) { - MutexUnlocker mul(Heap_lock); - GC_locker::stall_until_clear(); - - // We'll then fall off the end of the ("if GC locker active") - // if-statement and retry the allocation further down in the - // loop. - } else { + if (jthr->in_critical()) { if (CheckJNICalls) { fatal("Possible deadlock due to allocating while" " in jni critical section"); } + // We are returning NULL so the protocol is that we're still + // holding the Heap_lock. + assert_heap_locked(); return NULL; } + + Heap_lock->unlock(); + GC_locker::stall_until_clear(); + + // No need to relock the Heap_lock. We'll fall off to the code + // below the else-statement which assumes that we are not + // holding the Heap_lock. } else { // We are not locked out. So, let's try to do a GC. The VM op // will retry the allocation before it completes. @@ -805,11 +959,10 @@ dirty_young_block(result, word_size); return result; } - - Heap_lock->lock(); } - assert_heap_locked(); + // Both paths that get us here from above unlock the Heap_lock. + assert_heap_not_locked(); // We can reach here when we were unsuccessful in doing a GC, // because another thread beat us to it, or because we were locked @@ -854,7 +1007,7 @@ // allocation paths that attempt to allocate a humongous object // should eventually reach here. Currently, the only paths are from // mem_allocate() and attempt_allocation_at_safepoint(). - assert_heap_locked_or_at_safepoint(); + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); assert(isHumongous(word_size), "attempt_allocation_humongous() " "should only be used for humongous allocations"); assert(SafepointSynchronize::is_at_safepoint() == at_safepoint, @@ -931,13 +1084,13 @@ } } - assert_heap_locked_or_at_safepoint(); + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); return NULL; } HeapWord* G1CollectedHeap::attempt_allocation_at_safepoint(size_t word_size, bool expect_null_cur_alloc_region) { - assert_at_safepoint(); + assert_at_safepoint(true /* should_be_vm_thread */); assert(_cur_alloc_region == NULL || !expect_null_cur_alloc_region, err_msg("the current alloc region was unexpectedly found " "to be non-NULL, cur alloc region: "PTR_FORMAT" " @@ -948,10 +1101,8 @@ if (!expect_null_cur_alloc_region) { HeapRegion* cur_alloc_region = _cur_alloc_region; if (cur_alloc_region != NULL) { - // This allocate method does BOT updates and we don't need them in - // the young generation. This will be fixed in the near future by - // CR 6994297. - HeapWord* result = cur_alloc_region->allocate(word_size); + // We are at a safepoint so no reason to use the MT-safe version. + HeapWord* result = cur_alloc_region->allocate_no_bot_updates(word_size); if (result != NULL) { assert(is_in(result), "result should be in the heap"); @@ -983,20 +1134,17 @@ assert_heap_not_locked_and_not_at_safepoint(); assert(!isHumongous(word_size), "we do not allow TLABs of humongous size"); - Heap_lock->lock(); - - // First attempt: try allocating out of the current alloc region or - // after replacing the current alloc region. + // First attempt: Try allocating out of the current alloc region + // using a CAS. If that fails, take the Heap_lock and retry the + // allocation, potentially replacing the current alloc region. HeapWord* result = attempt_allocation(word_size); if (result != NULL) { assert_heap_not_locked(); return result; } - assert_heap_locked(); - - // Second attempt: go into the even slower path where we might - // try to schedule a collection. + // Second attempt: Go to the slower path where we might try to + // schedule a collection. result = attempt_allocation_slow(word_size); if (result != NULL) { assert_heap_not_locked(); @@ -1004,6 +1152,7 @@ } assert_heap_locked(); + // Need to unlock the Heap_lock before returning. Heap_lock->unlock(); return NULL; } @@ -1022,11 +1171,10 @@ for (int try_count = 1; /* we'll return */; try_count += 1) { unsigned int gc_count_before; { - Heap_lock->lock(); - if (!isHumongous(word_size)) { - // First attempt: try allocating out of the current alloc - // region or after replacing the current alloc region. + // First attempt: Try allocating out of the current alloc region + // using a CAS. If that fails, take the Heap_lock and retry the + // allocation, potentially replacing the current alloc region. HeapWord* result = attempt_allocation(word_size); if (result != NULL) { assert_heap_not_locked(); @@ -1035,14 +1183,17 @@ assert_heap_locked(); - // Second attempt: go into the even slower path where we might - // try to schedule a collection. + // Second attempt: Go to the slower path where we might try to + // schedule a collection. result = attempt_allocation_slow(word_size); if (result != NULL) { assert_heap_not_locked(); return result; } } else { + // attempt_allocation_humongous() requires the Heap_lock to be held. + Heap_lock->lock(); + HeapWord* result = attempt_allocation_humongous(word_size, false /* at_safepoint */); if (result != NULL) { @@ -1054,7 +1205,8 @@ assert_heap_locked(); // Read the gc count while the heap lock is held. gc_count_before = SharedHeap::heap()->total_collections(); - // We cannot be at a safepoint, so it is safe to unlock the Heap_lock + + // Release the Heap_lock before attempting the collection. Heap_lock->unlock(); } @@ -1092,22 +1244,18 @@ } void G1CollectedHeap::abandon_cur_alloc_region() { - if (_cur_alloc_region != NULL) { - // We're finished with the _cur_alloc_region. - if (_cur_alloc_region->is_empty()) { - _free_regions++; - free_region(_cur_alloc_region); - } else { - // As we're builing (at least the young portion) of the collection - // set incrementally we'll add the current allocation region to - // the collection set here. - if (_cur_alloc_region->is_young()) { - g1_policy()->add_region_to_incremental_cset_lhs(_cur_alloc_region); - } - _summary_bytes_used += _cur_alloc_region->used(); - } - _cur_alloc_region = NULL; - } + assert_at_safepoint(true /* should_be_vm_thread */); + + HeapRegion* cur_alloc_region = _cur_alloc_region; + if (cur_alloc_region != NULL) { + assert(!cur_alloc_region->is_empty(), + "the current alloc region can never be empty"); + assert(cur_alloc_region->is_young(), + "the current alloc region should be young"); + + retire_cur_alloc_region_common(cur_alloc_region); + } + assert(_cur_alloc_region == NULL, "post-condition"); } void G1CollectedHeap::abandon_gc_alloc_regions() { @@ -1188,19 +1336,20 @@ bool G1CollectedHeap::do_collection(bool explicit_gc, bool clear_all_soft_refs, size_t word_size) { + assert_at_safepoint(true /* should_be_vm_thread */); + if (GC_locker::check_active_before_gc()) { return false; } - DTraceGCProbeMarker gc_probe_marker(true /* full */); + SvcGCMarker sgcm(SvcGCMarker::FULL); ResourceMark rm; if (PrintHeapAtGC) { Universe::print_heap_before_gc(); } - assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); - assert(Thread::current() == VMThread::vm_thread(), "should be in vm thread"); + verify_region_sets_optional(); const bool do_clear_all_soft_refs = clear_all_soft_refs || collector_policy()->should_clear_all_soft_refs(); @@ -1223,6 +1372,9 @@ double start = os::elapsedTime(); g1_policy()->record_full_collection_start(); + wait_while_free_regions_coming(); + append_secondary_free_list_if_not_empty(); + gc_prologue(true); increment_total_collections(true /* full gc */); @@ -1235,7 +1387,6 @@ gclog_or_tty->print(" VerifyBeforeGC:"); Universe::verify(true); } - assert(regions_accounted_for(), "Region leakage!"); COMPILER2_PRESENT(DerivedPointerTable::clear()); @@ -1259,7 +1410,6 @@ assert(_cur_alloc_region == NULL, "Invariant."); g1_rem_set()->cleanupHRRS(); tear_down_region_lists(); - set_used_regions_to_need_zero_fill(); // We may have added regions to the current incremental collection // set between the last GC or pause and now. We need to clear the @@ -1294,9 +1444,7 @@ HandleMark hm; // Discard invalid handles created during gc G1MarkSweep::invoke_at_safepoint(ref_processor(), do_clear_all_soft_refs); } - // Because freeing humongous regions may have added some unclean - // regions, it is necessary to tear down again before rebuilding. - tear_down_region_lists(); + assert(free_regions() == 0, "we should not have added any free regions"); rebuild_region_lists(); _summary_bytes_used = recalculate_used(); @@ -1378,7 +1526,6 @@ JavaThread::dirty_card_queue_set().abandon_logs(); assert(!G1DeferredRSUpdate || (G1DeferredRSUpdate && (dirty_card_queue_set().completed_buffers_num() == 0)), "Should not be any"); - assert(regions_accounted_for(), "Region leakage!"); } if (g1_policy()->in_young_gc_mode()) { @@ -1392,6 +1539,8 @@ // Update the number of full collections that have been completed. increment_full_collections_completed(false /* concurrent */); + verify_region_sets_optional(); + if (PrintHeapAtGC) { Universe::print_heap_after_gc(); } @@ -1532,10 +1681,7 @@ HeapWord* G1CollectedHeap::satisfy_failed_allocation(size_t word_size, bool* succeeded) { - assert(SafepointSynchronize::is_at_safepoint(), - "satisfy_failed_allocation() should only be called at a safepoint"); - assert(Thread::current()->is_VM_thread(), - "satisfy_failed_allocation() should only be called by the VM thread"); + assert_at_safepoint(true /* should_be_vm_thread */); *succeeded = true; // Let's attempt the allocation first. @@ -1607,53 +1753,22 @@ // allocated block, or else "NULL". HeapWord* G1CollectedHeap::expand_and_allocate(size_t word_size) { - assert(SafepointSynchronize::is_at_safepoint(), - "expand_and_allocate() should only be called at a safepoint"); - assert(Thread::current()->is_VM_thread(), - "expand_and_allocate() should only be called by the VM thread"); + assert_at_safepoint(true /* should_be_vm_thread */); + + verify_region_sets_optional(); size_t expand_bytes = word_size * HeapWordSize; if (expand_bytes < MinHeapDeltaBytes) { expand_bytes = MinHeapDeltaBytes; } expand(expand_bytes); - assert(regions_accounted_for(), "Region leakage!"); + + verify_region_sets_optional(); return attempt_allocation_at_safepoint(word_size, false /* expect_null_cur_alloc_region */); } -size_t G1CollectedHeap::free_region_if_totally_empty(HeapRegion* hr) { - size_t pre_used = 0; - size_t cleared_h_regions = 0; - size_t freed_regions = 0; - UncleanRegionList local_list; - free_region_if_totally_empty_work(hr, pre_used, cleared_h_regions, - freed_regions, &local_list); - - finish_free_region_work(pre_used, cleared_h_regions, freed_regions, - &local_list); - return pre_used; -} - -void -G1CollectedHeap::free_region_if_totally_empty_work(HeapRegion* hr, - size_t& pre_used, - size_t& cleared_h, - size_t& freed_regions, - UncleanRegionList* list, - bool par) { - assert(!hr->continuesHumongous(), "should have filtered these out"); - size_t res = 0; - if (hr->used() > 0 && hr->garbage_bytes() == hr->used() && - !hr->is_young()) { - if (G1PolicyVerbose > 0) - gclog_or_tty->print_cr("Freeing empty region "PTR_FORMAT "(" SIZE_FORMAT " bytes)" - " during cleanup", hr, hr->used()); - free_region_work(hr, pre_used, cleared_h, freed_regions, list, par); - } -} - // FIXME: both this and shrink could probably be more efficient by // doing one "VirtualSpace::expand_by" call rather than several. void G1CollectedHeap::expand(size_t expand_bytes) { @@ -1686,19 +1801,7 @@ // Add it to the HeapRegionSeq. _hrs->insert(hr); - // Set the zero-fill state, according to whether it's already - // zeroed. - { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - if (is_zeroed) { - hr->set_zero_fill_complete(); - put_free_region_on_list_locked(hr); - } else { - hr->set_zero_fill_needed(); - put_region_on_unclean_list_locked(hr); - } - } - _free_regions++; + _free_list.add_as_tail(hr); // And we used up an expansion region to create it. _expansion_regions--; // Tell the cardtable about it. @@ -1707,6 +1810,7 @@ _bot_shared->resize(_g1_committed.word_size()); } } + if (Verbose && PrintGC) { size_t new_mem_size = _g1_storage.committed_size(); gclog_or_tty->print_cr("Expanding garbage-first heap from %ldK by %ldK to %ldK", @@ -1731,7 +1835,6 @@ assert(mr.start() == (HeapWord*)_g1_storage.high(), "Bad shrink!"); _g1_committed.set_end(mr.start()); - _free_regions -= num_regions_deleted; _expansion_regions += num_regions_deleted; // Tell the cardtable about it. @@ -1751,10 +1854,17 @@ } void G1CollectedHeap::shrink(size_t shrink_bytes) { + verify_region_sets_optional(); + release_gc_alloc_regions(true /* totally */); + // Instead of tearing down / rebuilding the free lists here, we + // could instead use the remove_all_pending() method on free_list to + // remove only the ones that we need to remove. tear_down_region_lists(); // We will rebuild them in a moment. shrink_helper(shrink_bytes); rebuild_region_lists(); + + verify_region_sets_optional(); } // Public methods. @@ -1773,18 +1883,17 @@ _ref_processor(NULL), _process_strong_tasks(new SubTasksDone(G1H_PS_NumElements)), _bot_shared(NULL), - _par_alloc_during_gc_lock(Mutex::leaf, "par alloc during GC lock"), _objs_with_preserved_marks(NULL), _preserved_marks_of_objs(NULL), _evac_failure_scan_stack(NULL) , _mark_in_progress(false), - _cg1r(NULL), _czft(NULL), _summary_bytes_used(0), + _cg1r(NULL), _summary_bytes_used(0), _cur_alloc_region(NULL), _refine_cte_cl(NULL), - _free_region_list(NULL), _free_region_list_size(0), - _free_regions(0), _full_collection(false), - _unclean_region_list(), - _unclean_regions_coming(false), + _free_list("Master Free List"), + _secondary_free_list("Secondary Free List"), + _humongous_set("Master Humongous Set"), + _free_regions_coming(false), _young_list(new YoungList(this)), _gc_time_stamp(0), _surviving_young_words(NULL), @@ -1868,7 +1977,7 @@ ReservedSpace heap_rs(max_byte_size + pgs->max_size(), HeapRegion::GrainBytes, - false /*ism*/, addr); + UseLargePages, addr); if (UseCompressedOops) { if (addr != NULL && !heap_rs.is_reserved()) { @@ -1877,13 +1986,13 @@ // Try again to reserver heap higher. addr = Universe::preferred_heap_base(total_reserved, Universe::ZeroBasedNarrowOop); ReservedSpace heap_rs0(total_reserved, HeapRegion::GrainBytes, - false /*ism*/, addr); + UseLargePages, addr); if (addr != NULL && !heap_rs0.is_reserved()) { // Failed to reserve at specified address again - give up. addr = Universe::preferred_heap_base(total_reserved, Universe::HeapBasedNarrowOop); assert(addr == NULL, ""); ReservedSpace heap_rs1(total_reserved, HeapRegion::GrainBytes, - false /*ism*/, addr); + UseLargePages, addr); heap_rs = heap_rs1; } else { heap_rs = heap_rs0; @@ -1905,8 +2014,6 @@ _expansion_regions = max_byte_size/HeapRegion::GrainBytes; - _num_humongous_regions = 0; - // Create the gen rem set (and barrier set) for the entire reserved region. _rem_set = collector_policy()->create_rem_set(_reserved, 2); set_barrier_set(rem_set()->bs()); @@ -1951,6 +2058,8 @@ guarantee((size_t) HeapRegion::CardsPerRegion < max_cards_per_region, "too many cards per region"); + HeapRegionSet::set_unrealistically_long_length(max_regions() + 1); + _bot_shared = new G1BlockOffsetSharedArray(_reserved, heap_word_size(init_byte_size)); @@ -1975,11 +2084,6 @@ _cm = new ConcurrentMark(heap_rs, (int) max_regions()); _cmThread = _cm->cmThread(); - // ...and the concurrent zero-fill thread, if necessary. - if (G1ConcZeroFill) { - _czft = new ConcurrentZFThread(); - } - // Initialize the from_card cache structure of HeapRegionRemSet. HeapRegionRemSet::init_heap(max_regions()); @@ -2153,7 +2257,7 @@ #endif // PRODUCT size_t G1CollectedHeap::unsafe_max_alloc() { - if (_free_regions > 0) return HeapRegion::GrainBytes; + if (free_regions() > 0) return HeapRegion::GrainBytes; // otherwise, is there space in the current allocation region? // We need to store the current allocation region in a local variable @@ -2233,8 +2337,7 @@ } void G1CollectedHeap::collect_as_vm_thread(GCCause::Cause cause) { - assert(Thread::current()->is_VM_thread(), "Precondition#1"); - assert(Heap_lock->is_locked(), "Precondition#2"); + assert_at_safepoint(true /* should_be_vm_thread */); GCCauseSetter gcs(this, cause); switch (cause) { case GCCause::_heap_inspection: @@ -2257,12 +2360,6 @@ { MutexLocker ml(Heap_lock); - // Don't want to do a GC until cleanup is completed. This - // limitation will be removed in the near future when the - // operation of the free region list is revamped as part of - // CR 6977804. - wait_for_cleanup_complete(); - // Read the GC count while holding the Heap_lock gc_count_before = SharedHeap::heap()->total_collections(); full_gc_count_before = SharedHeap::heap()->total_full_collections(); @@ -2641,10 +2738,6 @@ } } -bool G1CollectedHeap::allocs_are_zero_filled() { - return false; -} - size_t G1CollectedHeap::large_typearray_limit() { // FIXME return HeapRegion::GrainBytes/HeapWordSize; @@ -2659,7 +2752,6 @@ return 0; } - void G1CollectedHeap::prepare_for_verify() { if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) { ensure_parsability(false); @@ -2870,7 +2962,9 @@ &rootsCl); bool failures = rootsCl.failures(); rem_set()->invalidate(perm_gen()->used_region(), false); - if (!silent) { gclog_or_tty->print("heapRegions "); } + if (!silent) { gclog_or_tty->print("HeapRegionSets "); } + verify_region_sets(); + if (!silent) { gclog_or_tty->print("HeapRegions "); } if (GCParallelVerificationEnabled && ParallelGCThreads > 1) { assert(check_heap_region_claim_values(HeapRegion::InitialClaimValue), "sanity check"); @@ -2898,7 +2992,7 @@ failures = true; } } - if (!silent) gclog_or_tty->print("remset "); + if (!silent) gclog_or_tty->print("RemSet "); rem_set()->verify(); if (failures) { @@ -2969,15 +3063,10 @@ if (G1CollectedHeap::use_parallel_gc_threads()) { workers()->print_worker_threads_on(st); } - _cmThread->print_on(st); st->cr(); - _cm->print_worker_threads_on(st); - _cg1r->print_worker_threads_on(st); - - _czft->print_on(st); st->cr(); } @@ -2987,7 +3076,6 @@ } tc->do_thread(_cmThread); _cg1r->threads_do(tc); - tc->do_thread(_czft); } void G1CollectedHeap::print_tracing_info() const { @@ -3003,15 +3091,10 @@ if (G1SummarizeConcMark) { concurrent_mark()->print_summary_info(); } - if (G1SummarizeZFStats) { - ConcurrentZFThread::print_summary_info(); - } g1_policy()->print_yg_surv_rate_info(); - SpecializationStats::print(); } - int G1CollectedHeap::addr_to_arena_id(void* addr) const { HeapRegion* hr = heap_region_containing(addr); if (hr == NULL) { @@ -3210,17 +3293,22 @@ bool G1CollectedHeap::do_collection_pause_at_safepoint(double target_pause_time_ms) { + assert_at_safepoint(true /* should_be_vm_thread */); + guarantee(!is_gc_active(), "collection is not reentrant"); + if (GC_locker::check_active_before_gc()) { return false; } - DTraceGCProbeMarker gc_probe_marker(false /* full */); + SvcGCMarker sgcm(SvcGCMarker::MINOR); ResourceMark rm; if (PrintHeapAtGC) { Universe::print_heap_before_gc(); } + verify_region_sets_optional(); + { // This call will decide whether this pause is an initial-mark // pause. If it is, during_initial_mark_pause() will return true @@ -3251,10 +3339,16 @@ TraceMemoryManagerStats tms(false /* fullGC */); - assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); - assert(Thread::current() == VMThread::vm_thread(), "should be in vm thread"); - guarantee(!is_gc_active(), "collection is not reentrant"); - assert(regions_accounted_for(), "Region leakage!"); + // If there are any free regions available on the secondary_free_list + // make sure we append them to the free_list. However, we don't + // have to wait for the rest of the cleanup operation to + // finish. If it's still going on that's OK. If we run out of + // regions, the region allocation code will check the + // secondary_free_list and potentially wait if more free regions + // are coming (see new_region_try_secondary_free_list()). + if (!G1StressConcRegionFreeing) { + append_secondary_free_list_if_not_empty(); + } increment_gc_time_stamp(); @@ -3334,8 +3428,6 @@ // progress, this will be zero. _cm->set_oops_do_bound(); - assert(regions_accounted_for(), "Region leakage."); - if (mark_in_progress()) concurrent_mark()->newCSet(); @@ -3431,8 +3523,6 @@ g1_policy()->record_pause_time_ms(pause_time_ms); g1_policy()->record_collection_pause_end(); - assert(regions_accounted_for(), "Region leakage."); - MemoryService::track_memory_usage(); if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) { @@ -3463,8 +3553,6 @@ gc_epilogue(false); } - assert(verify_region_lists(), "Bad region lists."); - if (ExitAfterGCNum > 0 && total_collections() == ExitAfterGCNum) { gclog_or_tty->print_cr("Stopping after GC #%d", ExitAfterGCNum); print_tracing_info(); @@ -3472,6 +3560,8 @@ } } + verify_region_sets_optional(); + TASKQUEUE_STATS_ONLY(if (ParallelGCVerbose) print_taskqueue_stats()); TASKQUEUE_STATS_ONLY(reset_taskqueue_stats()); @@ -3578,7 +3668,7 @@ void G1CollectedHeap::push_gc_alloc_region(HeapRegion* hr) { assert(Thread::current()->is_VM_thread() || - par_alloc_during_gc_lock()->owned_by_self(), "Precondition"); + FreeList_lock->owned_by_self(), "Precondition"); assert(!hr->is_gc_alloc_region() && !hr->in_collection_set(), "Precondition."); hr->set_is_gc_alloc_region(true); @@ -3600,7 +3690,7 @@ #endif // G1_DEBUG void G1CollectedHeap::forget_alloc_region_list() { - assert(Thread::current()->is_VM_thread(), "Precondition"); + assert_at_safepoint(true /* should_be_vm_thread */); while (_gc_alloc_region_list != NULL) { HeapRegion* r = _gc_alloc_region_list; assert(r->is_gc_alloc_region(), "Invariant."); @@ -3620,9 +3710,6 @@ _young_list->add_survivor_region(r); } } - if (r->is_empty()) { - ++_free_regions; - } } #ifdef G1_DEBUG FindGCAllocRegion fa; @@ -3675,7 +3762,7 @@ if (alloc_region == NULL) { // we will get a new GC alloc region - alloc_region = newAllocRegionWithExpansion(ap, 0); + alloc_region = new_gc_alloc_region(ap, 0); } else { // the region was retained from the last collection ++_gc_alloc_region_counts[ap]; @@ -3730,11 +3817,9 @@ set_gc_alloc_region(ap, NULL); if (r->is_empty()) { - // we didn't actually allocate anything in it; let's just put - // it on the free list - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - r->set_zero_fill_complete(); - put_free_region_on_list_locked(r); + // We didn't actually allocate anything in it; let's just put + // it back on the free list. + _free_list.add_as_tail(r); } else if (_retain_gc_alloc_region[ap] && !totally) { // retain it so that we can use it at the beginning of the next GC _retained_gc_alloc_regions[ap] = r; @@ -3856,13 +3941,15 @@ size_t _next_marked_bytes; OopsInHeapRegionClosure *_cl; public: - RemoveSelfPointerClosure(G1CollectedHeap* g1, OopsInHeapRegionClosure* cl) : - _g1(g1), _cm(_g1->concurrent_mark()), _prev_marked_bytes(0), + RemoveSelfPointerClosure(G1CollectedHeap* g1, HeapRegion* hr, + OopsInHeapRegionClosure* cl) : + _g1(g1), _hr(hr), _cm(_g1->concurrent_mark()), _prev_marked_bytes(0), _next_marked_bytes(0), _cl(cl) {} size_t prev_marked_bytes() { return _prev_marked_bytes; } size_t next_marked_bytes() { return _next_marked_bytes; } + // // The original idea here was to coalesce evacuated and dead objects. // However that caused complications with the block offset table (BOT). // In particular if there were two TLABs, one of them partially refined. @@ -3871,15 +3958,24 @@ // of TLAB_2. If the last object of the TLAB_1 and the first object // of TLAB_2 are coalesced, then the cards of the unrefined part // would point into middle of the filler object. + // The current approach is to not coalesce and leave the BOT contents intact. + // // - // The current approach is to not coalesce and leave the BOT contents intact. + // We now reset the BOT when we start the object iteration over the + // region and refine its entries for every object we come across. So + // the above comment is not really relevant and we should be able + // to coalesce dead objects if we want to. void do_object(oop obj) { + HeapWord* obj_addr = (HeapWord*) obj; + assert(_hr->is_in(obj_addr), "sanity"); + size_t obj_size = obj->size(); + _hr->update_bot_for_object(obj_addr, obj_size); if (obj->is_forwarded() && obj->forwardee() == obj) { // The object failed to move. assert(!_g1->is_obj_dead(obj), "We should not be preserving dead objs."); _cm->markPrev(obj); assert(_cm->isPrevMarked(obj), "Should be marked!"); - _prev_marked_bytes += (obj->size() * HeapWordSize); + _prev_marked_bytes += (obj_size * HeapWordSize); if (_g1->mark_in_progress() && !_g1->is_obj_ill(obj)) { _cm->markAndGrayObjectIfNecessary(obj); } @@ -3901,7 +3997,7 @@ } else { // The object has been either evacuated or is dead. Fill it with a // dummy object. - MemRegion mr((HeapWord*)obj, obj->size()); + MemRegion mr((HeapWord*)obj, obj_size); CollectedHeap::fill_with_object(mr); _cm->clearRangeBothMaps(mr); } @@ -3921,10 +4017,13 @@ HeapRegion* cur = g1_policy()->collection_set(); while (cur != NULL) { assert(g1_policy()->assertMarkedBytesDataOK(), "Should be!"); - - RemoveSelfPointerClosure rspc(_g1h, cl); + assert(!cur->isHumongous(), "sanity"); + if (cur->evacuation_failed()) { assert(cur->in_collection_set(), "bad CS"); + RemoveSelfPointerClosure rspc(_g1h, cur, cl); + + cur->reset_bot(); cl->set_region(cur); cur->object_iterate(&rspc); @@ -3989,15 +4088,6 @@ } } -void G1CollectedHeap::handle_evacuation_failure(oop old) { - markOop m = old->mark(); - // forward to self - assert(!old->is_forwarded(), "precondition"); - - old->forward_to(old); - handle_evacuation_failure_common(old, m); -} - oop G1CollectedHeap::handle_evacuation_failure_par(OopsInHeapRegionClosure* cl, oop old) { @@ -4084,8 +4174,6 @@ HeapWord* block = alloc_region->par_allocate(word_size); if (block == NULL) { - MutexLockerEx x(par_alloc_during_gc_lock(), - Mutex::_no_safepoint_check_flag); block = allocate_during_gc_slow(purpose, alloc_region, true, word_size); } return block; @@ -4114,6 +4202,12 @@ err_msg("we should not be seeing humongous allocation requests " "during GC, word_size = "SIZE_FORMAT, word_size)); + // We need to make sure we serialize calls to this method. Given + // that the FreeList_lock guards accesses to the free_list anyway, + // and we need to potentially remove a region from it, we'll use it + // to protect the whole call. + MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag); + HeapWord* block = NULL; // In the parallel case, a previous thread to obtain the lock may have // already assigned a new gc_alloc_region. @@ -4159,7 +4253,7 @@ } // Now allocate a new region for allocation. - alloc_region = newAllocRegionWithExpansion(purpose, word_size, false /*zero_filled*/); + alloc_region = new_gc_alloc_region(purpose, word_size); // let the caller handle alloc failure if (alloc_region != NULL) { @@ -4167,9 +4261,6 @@ assert(check_gc_alloc_regions(), "alloc regions messed up"); assert(alloc_region->saved_mark_at_top(), "Mark should have been saved already."); - // We used to assert that the region was zero-filled here, but no - // longer. - // This must be done last: once it's installed, other regions may // allocate in it (without holding the lock.) set_gc_alloc_region(purpose, alloc_region); @@ -4834,91 +4925,91 @@ COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); } -void G1CollectedHeap::free_region(HeapRegion* hr) { - size_t pre_used = 0; - size_t cleared_h_regions = 0; - size_t freed_regions = 0; - UncleanRegionList local_list; - - HeapWord* start = hr->bottom(); - HeapWord* end = hr->prev_top_at_mark_start(); - size_t used_bytes = hr->used(); - size_t live_bytes = hr->max_live_bytes(); - if (used_bytes > 0) { - guarantee( live_bytes <= used_bytes, "invariant" ); - } else { - guarantee( live_bytes == 0, "invariant" ); - } - - size_t garbage_bytes = used_bytes - live_bytes; - if (garbage_bytes > 0) - g1_policy()->decrease_known_garbage_bytes(garbage_bytes); - - free_region_work(hr, pre_used, cleared_h_regions, freed_regions, - &local_list); - finish_free_region_work(pre_used, cleared_h_regions, freed_regions, - &local_list); -} - -void -G1CollectedHeap::free_region_work(HeapRegion* hr, - size_t& pre_used, - size_t& cleared_h_regions, - size_t& freed_regions, - UncleanRegionList* list, - bool par) { - pre_used += hr->used(); - if (hr->isHumongous()) { - assert(hr->startsHumongous(), - "Only the start of a humongous region should be freed."); - int ind = _hrs->find(hr); - assert(ind != -1, "Should have an index."); - // Clear the start region. - hr->hr_clear(par, true /*clear_space*/); - list->insert_before_head(hr); - cleared_h_regions++; - freed_regions++; - // Clear any continued regions. - ind++; - while ((size_t)ind < n_regions()) { - HeapRegion* hrc = _hrs->at(ind); - if (!hrc->continuesHumongous()) break; - // Otherwise, does continue the H region. - assert(hrc->humongous_start_region() == hr, "Huh?"); - hrc->hr_clear(par, true /*clear_space*/); - cleared_h_regions++; - freed_regions++; - list->insert_before_head(hrc); - ind++; +void G1CollectedHeap::free_region_if_totally_empty(HeapRegion* hr, + size_t* pre_used, + FreeRegionList* free_list, + HumongousRegionSet* humongous_proxy_set, + bool par) { + if (hr->used() > 0 && hr->max_live_bytes() == 0 && !hr->is_young()) { + if (hr->isHumongous()) { + assert(hr->startsHumongous(), "we should only see starts humongous"); + free_humongous_region(hr, pre_used, free_list, humongous_proxy_set, par); + } else { + free_region(hr, pre_used, free_list, par); } - } else { - hr->hr_clear(par, true /*clear_space*/); - list->insert_before_head(hr); - freed_regions++; - // If we're using clear2, this should not be enabled. - // assert(!hr->in_cohort(), "Can't be both free and in a cohort."); } } -void G1CollectedHeap::finish_free_region_work(size_t pre_used, - size_t cleared_h_regions, - size_t freed_regions, - UncleanRegionList* list) { - if (list != NULL && list->sz() > 0) { - prepend_region_list_on_unclean_list(list); - } - // Acquire a lock, if we're parallel, to update possibly-shared - // variables. - Mutex* lock = (n_par_threads() > 0) ? ParGCRareEvent_lock : NULL; - { +void G1CollectedHeap::free_region(HeapRegion* hr, + size_t* pre_used, + FreeRegionList* free_list, + bool par) { + assert(!hr->isHumongous(), "this is only for non-humongous regions"); + assert(!hr->is_empty(), "the region should not be empty"); + assert(free_list != NULL, "pre-condition"); + + *pre_used += hr->used(); + hr->hr_clear(par, true /* clear_space */); + free_list->add_as_tail(hr); +} + +void G1CollectedHeap::free_humongous_region(HeapRegion* hr, + size_t* pre_used, + FreeRegionList* free_list, + HumongousRegionSet* humongous_proxy_set, + bool par) { + assert(hr->startsHumongous(), "this is only for starts humongous regions"); + assert(free_list != NULL, "pre-condition"); + assert(humongous_proxy_set != NULL, "pre-condition"); + + size_t hr_used = hr->used(); + size_t hr_capacity = hr->capacity(); + size_t hr_pre_used = 0; + _humongous_set.remove_with_proxy(hr, humongous_proxy_set); + hr->set_notHumongous(); + free_region(hr, &hr_pre_used, free_list, par); + + int i = hr->hrs_index() + 1; + size_t num = 1; + while ((size_t) i < n_regions()) { + HeapRegion* curr_hr = _hrs->at(i); + if (!curr_hr->continuesHumongous()) { + break; + } + curr_hr->set_notHumongous(); + free_region(curr_hr, &hr_pre_used, free_list, par); + num += 1; + i += 1; + } + assert(hr_pre_used == hr_used, + err_msg("hr_pre_used: "SIZE_FORMAT" and hr_used: "SIZE_FORMAT" " + "should be the same", hr_pre_used, hr_used)); + *pre_used += hr_pre_used; +} + +void G1CollectedHeap::update_sets_after_freeing_regions(size_t pre_used, + FreeRegionList* free_list, + HumongousRegionSet* humongous_proxy_set, + bool par) { + if (pre_used > 0) { + Mutex* lock = (par) ? ParGCRareEvent_lock : NULL; MutexLockerEx x(lock, Mutex::_no_safepoint_check_flag); + assert(_summary_bytes_used >= pre_used, + err_msg("invariant: _summary_bytes_used: "SIZE_FORMAT" " + "should be >= pre_used: "SIZE_FORMAT, + _summary_bytes_used, pre_used)); _summary_bytes_used -= pre_used; - _num_humongous_regions -= (int) cleared_h_regions; - _free_regions += freed_regions; + } + if (free_list != NULL && !free_list->is_empty()) { + MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag); + _free_list.add_as_tail(free_list); + } + if (humongous_proxy_set != NULL && !humongous_proxy_set->is_empty()) { + MutexLockerEx x(OldSets_lock, Mutex::_no_safepoint_check_flag); + _humongous_set.update_from_proxy(humongous_proxy_set); } } - void G1CollectedHeap::dirtyCardsForYoungRegions(CardTableModRefBS* ct_bs, HeapRegion* list) { while (list != NULL) { guarantee( list->is_young(), "invariant" ); @@ -5041,6 +5132,9 @@ } void G1CollectedHeap::free_collection_set(HeapRegion* cs_head) { + size_t pre_used = 0; + FreeRegionList local_free_list("Local List for CSet Freeing"); + double young_time_ms = 0.0; double non_young_time_ms = 0.0; @@ -5059,6 +5153,8 @@ size_t rs_lengths = 0; while (cur != NULL) { + assert(!is_on_free_list(cur), "sanity"); + if (non_young) { if (cur->is_young()) { double end_sec = os::elapsedTime(); @@ -5069,14 +5165,12 @@ non_young = false; } } else { - if (!cur->is_on_free_list()) { - double end_sec = os::elapsedTime(); - double elapsed_ms = (end_sec - start_sec) * 1000.0; - young_time_ms += elapsed_ms; - - start_sec = os::elapsedTime(); - non_young = true; - } + double end_sec = os::elapsedTime(); + double elapsed_ms = (end_sec - start_sec) * 1000.0; + young_time_ms += elapsed_ms; + + start_sec = os::elapsedTime(); + non_young = true; } rs_lengths += cur->rem_set()->occupied(); @@ -5109,9 +5203,8 @@ if (!cur->evacuation_failed()) { // And the region is empty. - assert(!cur->is_empty(), - "Should not have empty regions in a CS."); - free_region(cur); + assert(!cur->is_empty(), "Should not have empty regions in a CS."); + free_region(cur, &pre_used, &local_free_list, false /* par */); } else { cur->uninstall_surv_rate_group(); if (cur->is_young()) @@ -5132,6 +5225,9 @@ else young_time_ms += elapsed_ms; + update_sets_after_freeing_regions(pre_used, &local_free_list, + NULL /* humongous_proxy_set */, + false /* par */); policy->record_young_free_cset_time_ms(young_time_ms); policy->record_non_young_free_cset_time_ms(non_young_time_ms); } @@ -5157,291 +5253,53 @@ } } -HeapRegion* -G1CollectedHeap::alloc_region_from_unclean_list_locked(bool zero_filled) { - assert(ZF_mon->owned_by_self(), "Precondition"); - HeapRegion* res = pop_unclean_region_list_locked(); - if (res != NULL) { - assert(!res->continuesHumongous() && - res->zero_fill_state() != HeapRegion::Allocated, - "Only free regions on unclean list."); - if (zero_filled) { - res->ensure_zero_filled_locked(); - res->set_zero_fill_allocated(); - } - } - return res; -} - -HeapRegion* G1CollectedHeap::alloc_region_from_unclean_list(bool zero_filled) { - MutexLockerEx zx(ZF_mon, Mutex::_no_safepoint_check_flag); - return alloc_region_from_unclean_list_locked(zero_filled); +void G1CollectedHeap::set_free_regions_coming() { + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [cm thread] : " + "setting free regions coming"); + } + + assert(!free_regions_coming(), "pre-condition"); + _free_regions_coming = true; } -void G1CollectedHeap::put_region_on_unclean_list(HeapRegion* r) { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - put_region_on_unclean_list_locked(r); - if (should_zf()) ZF_mon->notify_all(); // Wake up ZF thread. -} - -void G1CollectedHeap::set_unclean_regions_coming(bool b) { - MutexLockerEx x(Cleanup_mon); - set_unclean_regions_coming_locked(b); -} - -void G1CollectedHeap::set_unclean_regions_coming_locked(bool b) { - assert(Cleanup_mon->owned_by_self(), "Precondition"); - _unclean_regions_coming = b; - // Wake up mutator threads that might be waiting for completeCleanup to - // finish. - if (!b) Cleanup_mon->notify_all(); -} - -void G1CollectedHeap::wait_for_cleanup_complete() { - assert_not_at_safepoint(); - MutexLockerEx x(Cleanup_mon); - wait_for_cleanup_complete_locked(); -} - -void G1CollectedHeap::wait_for_cleanup_complete_locked() { - assert(Cleanup_mon->owned_by_self(), "precondition"); - while (_unclean_regions_coming) { - Cleanup_mon->wait(); +void G1CollectedHeap::reset_free_regions_coming() { + { + assert(free_regions_coming(), "pre-condition"); + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + _free_regions_coming = false; + SecondaryFreeList_lock->notify_all(); + } + + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [cm thread] : " + "reset free regions coming"); } } -void -G1CollectedHeap::put_region_on_unclean_list_locked(HeapRegion* r) { - assert(ZF_mon->owned_by_self(), "precondition."); -#ifdef ASSERT - if (r->is_gc_alloc_region()) { - ResourceMark rm; - stringStream region_str; - print_on(®ion_str); - assert(!r->is_gc_alloc_region(), err_msg("Unexpected GC allocation region: %s", - region_str.as_string())); - } -#endif - _unclean_region_list.insert_before_head(r); -} - -void -G1CollectedHeap::prepend_region_list_on_unclean_list(UncleanRegionList* list) { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - prepend_region_list_on_unclean_list_locked(list); - if (should_zf()) ZF_mon->notify_all(); // Wake up ZF thread. -} - -void -G1CollectedHeap:: -prepend_region_list_on_unclean_list_locked(UncleanRegionList* list) { - assert(ZF_mon->owned_by_self(), "precondition."); - _unclean_region_list.prepend_list(list); -} - -HeapRegion* G1CollectedHeap::pop_unclean_region_list_locked() { - assert(ZF_mon->owned_by_self(), "precondition."); - HeapRegion* res = _unclean_region_list.pop(); - if (res != NULL) { - // Inform ZF thread that there's a new unclean head. - if (_unclean_region_list.hd() != NULL && should_zf()) - ZF_mon->notify_all(); - } - return res; -} - -HeapRegion* G1CollectedHeap::peek_unclean_region_list_locked() { - assert(ZF_mon->owned_by_self(), "precondition."); - return _unclean_region_list.hd(); -} - - -bool G1CollectedHeap::move_cleaned_region_to_free_list_locked() { - assert(ZF_mon->owned_by_self(), "Precondition"); - HeapRegion* r = peek_unclean_region_list_locked(); - if (r != NULL && r->zero_fill_state() == HeapRegion::ZeroFilled) { - // Result of below must be equal to "r", since we hold the lock. - (void)pop_unclean_region_list_locked(); - put_free_region_on_list_locked(r); - return true; - } else { - return false; - } -} - -bool G1CollectedHeap::move_cleaned_region_to_free_list() { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - return move_cleaned_region_to_free_list_locked(); -} - - -void G1CollectedHeap::put_free_region_on_list_locked(HeapRegion* r) { - assert(ZF_mon->owned_by_self(), "precondition."); - assert(_free_region_list_size == free_region_list_length(), "Inv"); - assert(r->zero_fill_state() == HeapRegion::ZeroFilled, - "Regions on free list must be zero filled"); - assert(!r->isHumongous(), "Must not be humongous."); - assert(r->is_empty(), "Better be empty"); - assert(!r->is_on_free_list(), - "Better not already be on free list"); - assert(!r->is_on_unclean_list(), - "Better not already be on unclean list"); - r->set_on_free_list(true); - r->set_next_on_free_list(_free_region_list); - _free_region_list = r; - _free_region_list_size++; - assert(_free_region_list_size == free_region_list_length(), "Inv"); -} - -void G1CollectedHeap::put_free_region_on_list(HeapRegion* r) { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - put_free_region_on_list_locked(r); -} - -HeapRegion* G1CollectedHeap::pop_free_region_list_locked() { - assert(ZF_mon->owned_by_self(), "precondition."); - assert(_free_region_list_size == free_region_list_length(), "Inv"); - HeapRegion* res = _free_region_list; - if (res != NULL) { - _free_region_list = res->next_from_free_list(); - _free_region_list_size--; - res->set_on_free_list(false); - res->set_next_on_free_list(NULL); - assert(_free_region_list_size == free_region_list_length(), "Inv"); - } - return res; -} - - -HeapRegion* G1CollectedHeap::alloc_free_region_from_lists(bool zero_filled) { - // By self, or on behalf of self. - assert(Heap_lock->is_locked(), "Precondition"); - HeapRegion* res = NULL; - bool first = true; - while (res == NULL) { - if (zero_filled || !first) { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - res = pop_free_region_list_locked(); - if (res != NULL) { - assert(!res->zero_fill_is_allocated(), - "No allocated regions on free list."); - res->set_zero_fill_allocated(); - } else if (!first) { - break; // We tried both, time to return NULL. - } - } - - if (res == NULL) { - res = alloc_region_from_unclean_list(zero_filled); - } - assert(res == NULL || - !zero_filled || - res->zero_fill_is_allocated(), - "We must have allocated the region we're returning"); - first = false; - } - return res; -} - -void G1CollectedHeap::remove_allocated_regions_from_lists() { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - { - HeapRegion* prev = NULL; - HeapRegion* cur = _unclean_region_list.hd(); - while (cur != NULL) { - HeapRegion* next = cur->next_from_unclean_list(); - if (cur->zero_fill_is_allocated()) { - // Remove from the list. - if (prev == NULL) { - (void)_unclean_region_list.pop(); - } else { - _unclean_region_list.delete_after(prev); - } - cur->set_on_unclean_list(false); - cur->set_next_on_unclean_list(NULL); - } else { - prev = cur; - } - cur = next; - } - assert(_unclean_region_list.sz() == unclean_region_list_length(), - "Inv"); +void G1CollectedHeap::wait_while_free_regions_coming() { + // Most of the time we won't have to wait, so let's do a quick test + // first before we take the lock. + if (!free_regions_coming()) { + return; + } + + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [other] : " + "waiting for free regions"); } { - HeapRegion* prev = NULL; - HeapRegion* cur = _free_region_list; - while (cur != NULL) { - HeapRegion* next = cur->next_from_free_list(); - if (cur->zero_fill_is_allocated()) { - // Remove from the list. - if (prev == NULL) { - _free_region_list = cur->next_from_free_list(); - } else { - prev->set_next_on_free_list(cur->next_from_free_list()); - } - cur->set_on_free_list(false); - cur->set_next_on_free_list(NULL); - _free_region_list_size--; - } else { - prev = cur; - } - cur = next; + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + while (free_regions_coming()) { + SecondaryFreeList_lock->wait(Mutex::_no_safepoint_check_flag); } - assert(_free_region_list_size == free_region_list_length(), "Inv"); - } -} - -bool G1CollectedHeap::verify_region_lists() { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - return verify_region_lists_locked(); -} - -bool G1CollectedHeap::verify_region_lists_locked() { - HeapRegion* unclean = _unclean_region_list.hd(); - while (unclean != NULL) { - guarantee(unclean->is_on_unclean_list(), "Well, it is!"); - guarantee(!unclean->is_on_free_list(), "Well, it shouldn't be!"); - guarantee(unclean->zero_fill_state() != HeapRegion::Allocated, - "Everything else is possible."); - unclean = unclean->next_from_unclean_list(); - } - guarantee(_unclean_region_list.sz() == unclean_region_list_length(), "Inv"); - - HeapRegion* free_r = _free_region_list; - while (free_r != NULL) { - assert(free_r->is_on_free_list(), "Well, it is!"); - assert(!free_r->is_on_unclean_list(), "Well, it shouldn't be!"); - switch (free_r->zero_fill_state()) { - case HeapRegion::NotZeroFilled: - case HeapRegion::ZeroFilling: - guarantee(false, "Should not be on free list."); - break; - default: - // Everything else is possible. - break; - } - free_r = free_r->next_from_free_list(); - } - guarantee(_free_region_list_size == free_region_list_length(), "Inv"); - // If we didn't do an assertion... - return true; -} - -size_t G1CollectedHeap::free_region_list_length() { - assert(ZF_mon->owned_by_self(), "precondition."); - size_t len = 0; - HeapRegion* cur = _free_region_list; - while (cur != NULL) { - len++; - cur = cur->next_from_free_list(); - } - return len; -} - -size_t G1CollectedHeap::unclean_region_list_length() { - assert(ZF_mon->owned_by_self(), "precondition."); - return _unclean_region_list.length(); + } + + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [other] : " + "done waiting for free regions"); + } } size_t G1CollectedHeap::n_regions() { @@ -5454,55 +5312,6 @@ HeapRegion::GrainBytes; } -size_t G1CollectedHeap::free_regions() { - /* Possibly-expensive assert. - assert(_free_regions == count_free_regions(), - "_free_regions is off."); - */ - return _free_regions; -} - -bool G1CollectedHeap::should_zf() { - return _free_region_list_size < (size_t) G1ConcZFMaxRegions; -} - -class RegionCounter: public HeapRegionClosure { - size_t _n; -public: - RegionCounter() : _n(0) {} - bool doHeapRegion(HeapRegion* r) { - if (r->is_empty()) { - assert(!r->isHumongous(), "H regions should not be empty."); - _n++; - } - return false; - } - int res() { return (int) _n; } -}; - -size_t G1CollectedHeap::count_free_regions() { - RegionCounter rc; - heap_region_iterate(&rc); - size_t n = rc.res(); - if (_cur_alloc_region != NULL && _cur_alloc_region->is_empty()) - n--; - return n; -} - -size_t G1CollectedHeap::count_free_regions_list() { - size_t n = 0; - size_t o = 0; - ZF_mon->lock_without_safepoint_check(); - HeapRegion* cur = _free_region_list; - while (cur != NULL) { - cur = cur->next_from_free_list(); - n++; - } - size_t m = unclean_region_list_length(); - ZF_mon->unlock(); - return n + m; -} - void G1CollectedHeap::set_region_short_lived_locked(HeapRegion* hr) { assert(heap_lock_held_for_gc(), "the heap lock should already be held by or for this thread"); @@ -5574,28 +5383,19 @@ } } - // Done at the start of full GC. void G1CollectedHeap::tear_down_region_lists() { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - while (pop_unclean_region_list_locked() != NULL) ; - assert(_unclean_region_list.hd() == NULL && _unclean_region_list.sz() == 0, - "Postconditions of loop."); - while (pop_free_region_list_locked() != NULL) ; - assert(_free_region_list == NULL, "Postcondition of loop."); - if (_free_region_list_size != 0) { - gclog_or_tty->print_cr("Size is %d.", _free_region_list_size); - print_on(gclog_or_tty, true /* extended */); - } - assert(_free_region_list_size == 0, "Postconditions of loop."); + _free_list.remove_all(); } - class RegionResetter: public HeapRegionClosure { - G1CollectedHeap* _g1; - int _n; + G1CollectedHeap* _g1h; + FreeRegionList _local_free_list; + public: - RegionResetter() : _g1(G1CollectedHeap::heap()), _n(0) {} + RegionResetter() : _g1h(G1CollectedHeap::heap()), + _local_free_list("Local Free List for RegionResetter") { } + bool doHeapRegion(HeapRegion* r) { if (r->continuesHumongous()) return false; if (r->top() > r->bottom()) { @@ -5603,152 +5403,32 @@ Copy::fill_to_words(r->top(), pointer_delta(r->end(), r->top())); } - r->set_zero_fill_allocated(); } else { assert(r->is_empty(), "tautology"); - _n++; - switch (r->zero_fill_state()) { - case HeapRegion::NotZeroFilled: - case HeapRegion::ZeroFilling: - _g1->put_region_on_unclean_list_locked(r); - break; - case HeapRegion::Allocated: - r->set_zero_fill_complete(); - // no break; go on to put on free list. - case HeapRegion::ZeroFilled: - _g1->put_free_region_on_list_locked(r); - break; - } + _local_free_list.add_as_tail(r); } return false; } - int getFreeRegionCount() {return _n;} + void update_free_lists() { + _g1h->update_sets_after_freeing_regions(0, &_local_free_list, NULL, + false /* par */); + } }; // Done at the end of full GC. void G1CollectedHeap::rebuild_region_lists() { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); // This needs to go at the end of the full GC. RegionResetter rs; heap_region_iterate(&rs); - _free_regions = rs.getFreeRegionCount(); - // Tell the ZF thread it may have work to do. - if (should_zf()) ZF_mon->notify_all(); -} - -class UsedRegionsNeedZeroFillSetter: public HeapRegionClosure { - G1CollectedHeap* _g1; - int _n; -public: - UsedRegionsNeedZeroFillSetter() : _g1(G1CollectedHeap::heap()), _n(0) {} - bool doHeapRegion(HeapRegion* r) { - if (r->continuesHumongous()) return false; - if (r->top() > r->bottom()) { - // There are assertions in "set_zero_fill_needed()" below that - // require top() == bottom(), so this is technically illegal. - // We'll skirt the law here, by making that true temporarily. - DEBUG_ONLY(HeapWord* save_top = r->top(); - r->set_top(r->bottom())); - r->set_zero_fill_needed(); - DEBUG_ONLY(r->set_top(save_top)); - } - return false; - } -}; - -// Done at the start of full GC. -void G1CollectedHeap::set_used_regions_to_need_zero_fill() { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - // This needs to go at the end of the full GC. - UsedRegionsNeedZeroFillSetter rs; - heap_region_iterate(&rs); + rs.update_free_lists(); } void G1CollectedHeap::set_refine_cte_cl_concurrency(bool concurrent) { _refine_cte_cl->set_concurrent(concurrent); } -#ifndef PRODUCT - -class PrintHeapRegionClosure: public HeapRegionClosure { -public: - bool doHeapRegion(HeapRegion *r) { - gclog_or_tty->print("Region: "PTR_FORMAT":", r); - if (r != NULL) { - if (r->is_on_free_list()) - gclog_or_tty->print("Free "); - if (r->is_young()) - gclog_or_tty->print("Young "); - if (r->isHumongous()) - gclog_or_tty->print("Is Humongous "); - r->print(); - } - return false; - } -}; - -class SortHeapRegionClosure : public HeapRegionClosure { - size_t young_regions,free_regions, unclean_regions; - size_t hum_regions, count; - size_t unaccounted, cur_unclean, cur_alloc; - size_t total_free; - HeapRegion* cur; -public: - SortHeapRegionClosure(HeapRegion *_cur) : cur(_cur), young_regions(0), - free_regions(0), unclean_regions(0), - hum_regions(0), - count(0), unaccounted(0), - cur_alloc(0), total_free(0) - {} - bool doHeapRegion(HeapRegion *r) { - count++; - if (r->is_on_free_list()) free_regions++; - else if (r->is_on_unclean_list()) unclean_regions++; - else if (r->isHumongous()) hum_regions++; - else if (r->is_young()) young_regions++; - else if (r == cur) cur_alloc++; - else unaccounted++; - return false; - } - void print() { - total_free = free_regions + unclean_regions; - gclog_or_tty->print("%d regions\n", count); - gclog_or_tty->print("%d free: free_list = %d unclean = %d\n", - total_free, free_regions, unclean_regions); - gclog_or_tty->print("%d humongous %d young\n", - hum_regions, young_regions); - gclog_or_tty->print("%d cur_alloc\n", cur_alloc); - gclog_or_tty->print("UHOH unaccounted = %d\n", unaccounted); - } -}; - -void G1CollectedHeap::print_region_counts() { - SortHeapRegionClosure sc(_cur_alloc_region); - PrintHeapRegionClosure cl; - heap_region_iterate(&cl); - heap_region_iterate(&sc); - sc.print(); - print_region_accounting_info(); -}; - -bool G1CollectedHeap::regions_accounted_for() { - // TODO: regions accounting for young/survivor/tenured - return true; -} - -bool G1CollectedHeap::print_region_accounting_info() { - gclog_or_tty->print_cr("Free regions: %d (count: %d count list %d) (clean: %d unclean: %d).", - free_regions(), - count_free_regions(), count_free_regions_list(), - _free_region_list_size, _unclean_region_list.sz()); - gclog_or_tty->print_cr("cur_alloc: %d.", - (_cur_alloc_region == NULL ? 0 : 1)); - gclog_or_tty->print_cr("H regions: %d.", _num_humongous_regions); - - // TODO: check regions accounting for young/survivor/tenured - return true; -} +#ifdef ASSERT bool G1CollectedHeap::is_in_closed_subset(const void* p) const { HeapRegion* hr = heap_region_containing(p); @@ -5758,8 +5438,84 @@ return hr->is_in(p); } } -#endif // !PRODUCT - -void G1CollectedHeap::g1_unimplemented() { - // Unimplemented(); +#endif // ASSERT + +class VerifyRegionListsClosure : public HeapRegionClosure { +private: + HumongousRegionSet* _humongous_set; + FreeRegionList* _free_list; + size_t _region_count; + +public: + VerifyRegionListsClosure(HumongousRegionSet* humongous_set, + FreeRegionList* free_list) : + _humongous_set(humongous_set), _free_list(free_list), + _region_count(0) { } + + size_t region_count() { return _region_count; } + + bool doHeapRegion(HeapRegion* hr) { + _region_count += 1; + + if (hr->continuesHumongous()) { + return false; + } + + if (hr->is_young()) { + // TODO + } else if (hr->startsHumongous()) { + _humongous_set->verify_next_region(hr); + } else if (hr->is_empty()) { + _free_list->verify_next_region(hr); + } + return false; + } +}; + +void G1CollectedHeap::verify_region_sets() { + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); + + // First, check the explicit lists. + _free_list.verify(); + { + // Given that a concurrent operation might be adding regions to + // the secondary free list we have to take the lock before + // verifying it. + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + _secondary_free_list.verify(); + } + _humongous_set.verify(); + + // If a concurrent region freeing operation is in progress it will + // be difficult to correctly attributed any free regions we come + // across to the correct free list given that they might belong to + // one of several (free_list, secondary_free_list, any local lists, + // etc.). So, if that's the case we will skip the rest of the + // verification operation. Alternatively, waiting for the concurrent + // operation to complete will have a non-trivial effect on the GC's + // operation (no concurrent operation will last longer than the + // interval between two calls to verification) and it might hide + // any issues that we would like to catch during testing. + if (free_regions_coming()) { + return; + } + + { + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + // Make sure we append the secondary_free_list on the free_list so + // that all free regions we will come across can be safely + // attributed to the free_list. + append_secondary_free_list(); + } + + // Finally, make sure that the region accounting in the lists is + // consistent with what we see in the heap. + _humongous_set.verify_start(); + _free_list.verify_start(); + + VerifyRegionListsClosure cl(&_humongous_set, &_free_list); + heap_region_iterate(&cl); + + _humongous_set.verify_end(); + _free_list.verify_end(); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp --- a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,7 +27,7 @@ #include "gc_implementation/g1/concurrentMark.hpp" #include "gc_implementation/g1/g1RemSet.hpp" -#include "gc_implementation/g1/heapRegion.hpp" +#include "gc_implementation/g1/heapRegionSets.hpp" #include "gc_implementation/parNew/parGCAllocBuffer.hpp" #include "memory/barrierSet.hpp" #include "memory/memRegion.hpp" @@ -66,8 +66,7 @@ enum G1GCThreadGroups { G1CRGroup = 0, G1ZFGroup = 1, - G1CMGroup = 2, - G1CLGroup = 3 + G1CMGroup = 2 }; enum GCAllocPurpose { @@ -155,6 +154,7 @@ friend class RefineCardTableEntryClosure; friend class G1PrepareCompactClosure; friend class RegionSorter; + friend class RegionResetter; friend class CountRCClosure; friend class EvacPopObjClosure; friend class G1ParCleanupCTTask; @@ -178,17 +178,20 @@ // The maximum part of _g1_storage that has ever been committed. MemRegion _g1_max_committed; - // The number of regions that are completely free. - size_t _free_regions; + // The master free list. It will satisfy all new region allocations. + MasterFreeRegionList _free_list; + + // The secondary free list which contains regions that have been + // freed up during the cleanup process. This will be appended to the + // master free list when appropriate. + SecondaryFreeRegionList _secondary_free_list; + + // It keeps track of the humongous regions. + MasterHumongousRegionSet _humongous_set; // The number of regions we could create by expansion. size_t _expansion_regions; - // Return the number of free regions in the heap (by direct counting.) - size_t count_free_regions(); - // Return the number of free regions on the free and unclean lists. - size_t count_free_regions_list(); - // The block offset table for the G1 heap. G1BlockOffsetSharedArray* _bot_shared; @@ -196,9 +199,6 @@ // lists, before and after full GC. void tear_down_region_lists(); void rebuild_region_lists(); - // This sets all non-empty regions to need zero-fill (which they will if - // they are empty after full collection.) - void set_used_regions_to_need_zero_fill(); // The sequence of all heap regions in the heap. HeapRegionSeq* _hrs; @@ -231,7 +231,7 @@ // Determines PLAB size for a particular allocation purpose. static size_t desired_plab_sz(GCAllocPurpose purpose); - // When called by par thread, require par_alloc_during_gc_lock() to be held. + // When called by par thread, requires the FreeList_lock to be held. void push_gc_alloc_region(HeapRegion* hr); // This should only be called single-threaded. Undeclares all GC alloc @@ -294,10 +294,11 @@ // line number, file, etc. #define heap_locking_asserts_err_msg(__extra_message) \ - err_msg("%s : Heap_lock %slocked, %sat a safepoint", \ + err_msg("%s : Heap_lock locked: %s, at safepoint: %s, is VM thread: %s", \ (__extra_message), \ - (!Heap_lock->owned_by_self()) ? "NOT " : "", \ - (!SafepointSynchronize::is_at_safepoint()) ? "NOT " : "") + BOOL_TO_STR(Heap_lock->owned_by_self()), \ + BOOL_TO_STR(SafepointSynchronize::is_at_safepoint()), \ + BOOL_TO_STR(Thread::current()->is_VM_thread())) #define assert_heap_locked() \ do { \ @@ -305,10 +306,11 @@ heap_locking_asserts_err_msg("should be holding the Heap_lock")); \ } while (0) -#define assert_heap_locked_or_at_safepoint() \ +#define assert_heap_locked_or_at_safepoint(__should_be_vm_thread) \ do { \ assert(Heap_lock->owned_by_self() || \ - SafepointSynchronize::is_at_safepoint(), \ + (SafepointSynchronize::is_at_safepoint() && \ + ((__should_be_vm_thread) == Thread::current()->is_VM_thread())), \ heap_locking_asserts_err_msg("should be holding the Heap_lock or " \ "should be at a safepoint")); \ } while (0) @@ -335,9 +337,10 @@ "should not be at a safepoint")); \ } while (0) -#define assert_at_safepoint() \ +#define assert_at_safepoint(__should_be_vm_thread) \ do { \ - assert(SafepointSynchronize::is_at_safepoint(), \ + assert(SafepointSynchronize::is_at_safepoint() && \ + ((__should_be_vm_thread) == Thread::current()->is_VM_thread()), \ heap_locking_asserts_err_msg("should be at a safepoint")); \ } while (0) @@ -362,31 +365,41 @@ // The current policy object for the collector. G1CollectorPolicy* _g1_policy; - // Parallel allocation lock to protect the current allocation region. - Mutex _par_alloc_during_gc_lock; - Mutex* par_alloc_during_gc_lock() { return &_par_alloc_during_gc_lock; } + // This is the second level of trying to allocate a new region. If + // new_region_work didn't find a region in the free_list, this call + // will check whether there's anything available in the + // secondary_free_list and/or wait for more regions to appear in that + // list, if _free_regions_coming is set. + HeapRegion* new_region_try_secondary_free_list(size_t word_size); - // If possible/desirable, allocate a new HeapRegion for normal object - // allocation sufficient for an allocation of the given "word_size". - // If "do_expand" is true, will attempt to expand the heap if necessary - // to to satisfy the request. If "zero_filled" is true, requires a - // zero-filled region. - // (Returning NULL will trigger a GC.) - virtual HeapRegion* newAllocRegion_work(size_t word_size, - bool do_expand, - bool zero_filled); + // It will try to allocate a single non-humongous HeapRegion + // sufficient for an allocation of the given word_size. If + // do_expand is true, it will attempt to expand the heap if + // necessary to satisfy the allocation request. Note that word_size + // is only used to make sure that we expand sufficiently but, given + // that the allocation request is assumed not to be humongous, + // having word_size is not strictly necessary (expanding by a single + // region will always be sufficient). But let's keep that parameter + // in case we need it in the future. + HeapRegion* new_region_work(size_t word_size, bool do_expand); - virtual HeapRegion* newAllocRegion(size_t word_size, - bool zero_filled = true) { - return newAllocRegion_work(word_size, false, zero_filled); + // It will try to allocate a new region to be used for allocation by + // mutator threads. It will not try to expand the heap if not region + // is available. + HeapRegion* new_alloc_region(size_t word_size) { + return new_region_work(word_size, false /* do_expand */); } - virtual HeapRegion* newAllocRegionWithExpansion(int purpose, - size_t word_size, - bool zero_filled = true); + + // It will try to allocate a new region to be used for allocation by + // a GC thread. It will try to expand the heap if no region is + // available. + HeapRegion* new_gc_alloc_region(int purpose, size_t word_size); + + int humongous_obj_allocate_find_first(size_t num_regions, size_t word_size); // Attempt to allocate an object of the given (very large) "word_size". // Returns "NULL" on failure. - virtual HeapWord* humongous_obj_allocate(size_t word_size); + HeapWord* humongous_obj_allocate(size_t word_size); // The following two methods, allocate_new_tlab() and // mem_allocate(), are the two main entry points from the runtime @@ -430,7 +443,8 @@ bool* gc_overhead_limit_was_exceeded); // The following methods, allocate_from_cur_allocation_region(), - // attempt_allocation(), replace_cur_alloc_region_and_allocate(), + // attempt_allocation(), attempt_allocation_locked(), + // replace_cur_alloc_region_and_allocate(), // attempt_allocation_slow(), and attempt_allocation_humongous() // have very awkward pre- and post-conditions with respect to // locking: @@ -481,20 +495,30 @@ // successfully manage to allocate it, or NULL. // It tries to satisfy an allocation request out of the current - // allocating region, which is passed as a parameter. It assumes - // that the caller has checked that the current allocating region is - // not NULL. Given that the caller has to check the current - // allocating region for at least NULL, it might as well pass it as - // the first parameter so that the method doesn't have to read it - // from the _cur_alloc_region field again. + // alloc region, which is passed as a parameter. It assumes that the + // caller has checked that the current alloc region is not NULL. + // Given that the caller has to check the current alloc region for + // at least NULL, it might as well pass it as the first parameter so + // that the method doesn't have to read it from the + // _cur_alloc_region field again. It is called from both + // attempt_allocation() and attempt_allocation_locked() and the + // with_heap_lock parameter indicates whether the caller was holding + // the heap lock when it called it or not. inline HeapWord* allocate_from_cur_alloc_region(HeapRegion* cur_alloc_region, - size_t word_size); + size_t word_size, + bool with_heap_lock); - // It attempts to allocate out of the current alloc region. If that - // fails, it retires the current alloc region (if there is one), - // tries to get a new one and retries the allocation. + // First-level of allocation slow path: it attempts to allocate out + // of the current alloc region in a lock-free manner using a CAS. If + // that fails it takes the Heap_lock and calls + // attempt_allocation_locked() for the second-level slow path. inline HeapWord* attempt_allocation(size_t word_size); + // Second-level of allocation slow path: while holding the Heap_lock + // it tries to allocate out of the current alloc region and, if that + // fails, tries to allocate out of a new current alloc region. + inline HeapWord* attempt_allocation_locked(size_t word_size); + // It assumes that the current alloc region has been retired and // tries to allocate a new one. If it's successful, it performs the // allocation out of the new current alloc region and updates @@ -506,11 +530,11 @@ bool do_dirtying, bool can_expand); - // The slow path when we are unable to allocate a new current alloc - // region to satisfy an allocation request (i.e., when - // attempt_allocation() fails). It will try to do an evacuation - // pause, which might stall due to the GC locker, and retry the - // allocation attempt when appropriate. + // Third-level of allocation slow path: when we are unable to + // allocate a new current alloc region to satisfy an allocation + // request (i.e., when attempt_allocation_locked() fails). It will + // try to do an evacuation pause, which might stall due to the GC + // locker, and retry the allocation attempt when appropriate. HeapWord* attempt_allocation_slow(size_t word_size); // The method that tries to satisfy a humongous allocation @@ -749,20 +773,29 @@ // Invoke "save_marks" on all heap regions. void save_marks(); - // Free a heap region. - void free_region(HeapRegion* hr); - // A component of "free_region", exposed for 'batching'. - // All the params after "hr" are out params: the used bytes of the freed - // region(s), the number of H regions cleared, the number of regions - // freed, and pointers to the head and tail of a list of freed contig - // regions, linked throught the "next_on_unclean_list" field. - void free_region_work(HeapRegion* hr, - size_t& pre_used, - size_t& cleared_h, - size_t& freed_regions, - UncleanRegionList* list, - bool par = false); + // It frees a non-humongous region by initializing its contents and + // adding it to the free list that's passed as a parameter (this is + // usually a local list which will be appended to the master free + // list later). The used bytes of freed regions are accumulated in + // pre_used. If par is true, the region's RSet will not be freed + // up. The assumption is that this will be done later. + void free_region(HeapRegion* hr, + size_t* pre_used, + FreeRegionList* free_list, + bool par); + // It frees a humongous region by collapsing it into individual + // regions and calling free_region() for each of them. The freed + // regions will be added to the free list that's passed as a parameter + // (this is usually a local list which will be appended to the + // master free list later). The used bytes of freed regions are + // accumulated in pre_used. If par is true, the region's RSet will + // not be freed up. The assumption is that this will be done later. + void free_humongous_region(HeapRegion* hr, + size_t* pre_used, + FreeRegionList* free_list, + HumongousRegionSet* humongous_proxy_set, + bool par); // The concurrent marker (and the thread it runs in.) ConcurrentMark* _cm; @@ -772,9 +805,6 @@ // The concurrent refiner. ConcurrentG1Refine* _cg1r; - // The concurrent zero-fill thread. - ConcurrentZFThread* _czft; - // The parallel task queues RefToScanQueueSet *_task_queues; @@ -826,7 +856,6 @@ void finalize_for_evac_failure(); // An attempt to evacuate "obj" has failed; take necessary steps. - void handle_evacuation_failure(oop obj); oop handle_evacuation_failure_par(OopsInHeapRegionClosure* cl, oop obj); void handle_evacuation_failure_common(oop obj, markOop m); @@ -867,9 +896,7 @@ SubTasksDone* _process_strong_tasks; - // List of regions which require zero filling. - UncleanRegionList _unclean_region_list; - bool _unclean_regions_coming; + volatile bool _free_regions_coming; public: @@ -992,71 +1019,64 @@ size_t max_regions(); // The number of regions that are completely free. - size_t free_regions(); + size_t free_regions() { + return _free_list.length(); + } // The number of regions that are not completely free. size_t used_regions() { return n_regions() - free_regions(); } - // True iff the ZF thread should run. - bool should_zf(); - // The number of regions available for "regular" expansion. size_t expansion_regions() { return _expansion_regions; } -#ifndef PRODUCT - bool regions_accounted_for(); - bool print_region_accounting_info(); - void print_region_counts(); -#endif - - HeapRegion* alloc_region_from_unclean_list(bool zero_filled); - HeapRegion* alloc_region_from_unclean_list_locked(bool zero_filled); - - void put_region_on_unclean_list(HeapRegion* r); - void put_region_on_unclean_list_locked(HeapRegion* r); + // verify_region_sets() performs verification over the region + // lists. It will be compiled in the product code to be used when + // necessary (i.e., during heap verification). + void verify_region_sets(); - void prepend_region_list_on_unclean_list(UncleanRegionList* list); - void prepend_region_list_on_unclean_list_locked(UncleanRegionList* list); + // verify_region_sets_optional() is planted in the code for + // list verification in non-product builds (and it can be enabled in + // product builds by definning HEAP_REGION_SET_FORCE_VERIFY to be 1). +#if HEAP_REGION_SET_FORCE_VERIFY + void verify_region_sets_optional() { + verify_region_sets(); + } +#else // HEAP_REGION_SET_FORCE_VERIFY + void verify_region_sets_optional() { } +#endif // HEAP_REGION_SET_FORCE_VERIFY - void set_unclean_regions_coming(bool b); - void set_unclean_regions_coming_locked(bool b); - // Wait for cleanup to be complete. - void wait_for_cleanup_complete(); - // Like above, but assumes that the calling thread owns the Heap_lock. - void wait_for_cleanup_complete_locked(); - - // Return the head of the unclean list. - HeapRegion* peek_unclean_region_list_locked(); - // Remove and return the head of the unclean list. - HeapRegion* pop_unclean_region_list_locked(); +#ifdef ASSERT + bool is_on_free_list(HeapRegion* hr) { + return hr->containing_set() == &_free_list; + } - // List of regions which are zero filled and ready for allocation. - HeapRegion* _free_region_list; - // Number of elements on the free list. - size_t _free_region_list_size; + bool is_on_humongous_set(HeapRegion* hr) { + return hr->containing_set() == &_humongous_set; +} +#endif // ASSERT - // If the head of the unclean list is ZeroFilled, move it to the free - // list. - bool move_cleaned_region_to_free_list_locked(); - bool move_cleaned_region_to_free_list(); + // Wrapper for the region list operations that can be called from + // methods outside this class. - void put_free_region_on_list_locked(HeapRegion* r); - void put_free_region_on_list(HeapRegion* r); + void secondary_free_list_add_as_tail(FreeRegionList* list) { + _secondary_free_list.add_as_tail(list); + } - // Remove and return the head element of the free list. - HeapRegion* pop_free_region_list_locked(); + void append_secondary_free_list() { + _free_list.add_as_tail(&_secondary_free_list); + } - // If "zero_filled" is true, we first try the free list, then we try the - // unclean list, zero-filling the result. If "zero_filled" is false, we - // first try the unclean list, then the zero-filled list. - HeapRegion* alloc_free_region_from_lists(bool zero_filled); + void append_secondary_free_list_if_not_empty() { + if (!_secondary_free_list.is_empty()) { + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + append_secondary_free_list(); + } + } - // Verify the integrity of the region lists. - void remove_allocated_regions_from_lists(); - bool verify_region_lists(); - bool verify_region_lists_locked(); - size_t unclean_region_list_length(); - size_t free_region_list_length(); + void set_free_regions_coming(); + void reset_free_regions_coming(); + bool free_regions_coming() { return _free_regions_coming; } + void wait_while_free_regions_coming(); // Perform a collection of the heap; intended for use in implementing // "System.gc". This probably implies as full a collection as the @@ -1075,23 +1095,24 @@ // True iff a evacuation has failed in the most-recent collection. bool evacuation_failed() { return _evacuation_failed; } - // Free a region if it is totally full of garbage. Returns the number of - // bytes freed (0 ==> didn't free it). - size_t free_region_if_totally_empty(HeapRegion *hr); - void free_region_if_totally_empty_work(HeapRegion *hr, - size_t& pre_used, - size_t& cleared_h_regions, - size_t& freed_regions, - UncleanRegionList* list, - bool par = false); + // It will free a region if it has allocated objects in it that are + // all dead. It calls either free_region() or + // free_humongous_region() depending on the type of the region that + // is passed to it. + void free_region_if_totally_empty(HeapRegion* hr, + size_t* pre_used, + FreeRegionList* free_list, + HumongousRegionSet* humongous_proxy_set, + bool par); - // If we've done free region work that yields the given changes, update - // the relevant global variables. - void finish_free_region_work(size_t pre_used, - size_t cleared_h_regions, - size_t freed_regions, - UncleanRegionList* list); - + // It appends the free list to the master free list and updates the + // master humongous list according to the contents of the proxy + // list. It also adjusts the total used bytes according to pre_used + // (if par is true, it will do so by taking the ParGCRareEvent_lock). + void update_sets_after_freeing_regions(size_t pre_used, + FreeRegionList* free_list, + HumongousRegionSet* humongous_proxy_set, + bool par); // Returns "TRUE" iff "p" points into the allocated area of the heap. virtual bool is_in(const void* p) const; @@ -1304,8 +1325,6 @@ return true; } - virtual bool allocs_are_zero_filled(); - // The boundary between a "large" and "small" array of primitives, in // words. virtual size_t large_typearray_limit(); @@ -1536,13 +1555,6 @@ protected: size_t _max_heap_capacity; - -public: - // Temporary: call to mark things unimplemented for the G1 heap (e.g., - // MemoryService). In productization, we can make this assert false - // to catch such places (as well as searching for calls to this...) - static void g1_unimplemented(); - }; #define use_local_bitmaps 1 diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp --- a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,7 +28,7 @@ #include "gc_implementation/g1/concurrentMark.hpp" #include "gc_implementation/g1/g1CollectedHeap.hpp" #include "gc_implementation/g1/g1CollectorPolicy.hpp" -#include "gc_implementation/g1/heapRegionSeq.hpp" +#include "gc_implementation/g1/heapRegionSeq.inline.hpp" #include "utilities/taskqueue.hpp" // Inline functions for G1CollectedHeap @@ -63,10 +63,12 @@ // assumptions of this method (and other related ones). inline HeapWord* G1CollectedHeap::allocate_from_cur_alloc_region(HeapRegion* cur_alloc_region, - size_t word_size) { - assert_heap_locked_and_not_at_safepoint(); + size_t word_size, + bool with_heap_lock) { + assert_not_at_safepoint(); + assert(with_heap_lock == Heap_lock->owned_by_self(), + "with_heap_lock and Heap_lock->owned_by_self() should be a tautology"); assert(cur_alloc_region != NULL, "pre-condition of the method"); - assert(cur_alloc_region == _cur_alloc_region, "pre-condition of the method"); assert(cur_alloc_region->is_young(), "we only support young current alloc regions"); assert(!isHumongous(word_size), "allocate_from_cur_alloc_region() " @@ -76,20 +78,24 @@ assert(!cur_alloc_region->is_empty(), err_msg("region ["PTR_FORMAT","PTR_FORMAT"] should not be empty", cur_alloc_region->bottom(), cur_alloc_region->end())); - // This allocate method does BOT updates and we don't need them in - // the young generation. This will be fixed in the near future by - // CR 6994297. - HeapWord* result = cur_alloc_region->allocate(word_size); + HeapWord* result = cur_alloc_region->par_allocate_no_bot_updates(word_size); if (result != NULL) { assert(is_in(result), "result should be in the heap"); - Heap_lock->unlock(); + if (with_heap_lock) { + Heap_lock->unlock(); + } + assert_heap_not_locked(); // Do the dirtying after we release the Heap_lock. dirty_young_block(result, word_size); return result; } - assert_heap_locked(); + if (with_heap_lock) { + assert_heap_locked(); + } else { + assert_heap_not_locked(); + } return NULL; } @@ -97,26 +103,75 @@ // assumptions of this method (and other related ones). inline HeapWord* G1CollectedHeap::attempt_allocation(size_t word_size) { - assert_heap_locked_and_not_at_safepoint(); + assert_heap_not_locked_and_not_at_safepoint(); assert(!isHumongous(word_size), "attempt_allocation() should not be called " "for humongous allocation requests"); HeapRegion* cur_alloc_region = _cur_alloc_region; if (cur_alloc_region != NULL) { HeapWord* result = allocate_from_cur_alloc_region(cur_alloc_region, - word_size); + word_size, + false /* with_heap_lock */); + assert_heap_not_locked(); + if (result != NULL) { + return result; + } + } + + // Our attempt to allocate lock-free failed as the current + // allocation region is either NULL or full. So, we'll now take the + // Heap_lock and retry. + Heap_lock->lock(); + + HeapWord* result = attempt_allocation_locked(word_size); + if (result != NULL) { + assert_heap_not_locked(); + return result; + } + + assert_heap_locked(); + return NULL; +} + +inline void +G1CollectedHeap::retire_cur_alloc_region_common(HeapRegion* cur_alloc_region) { + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); + assert(cur_alloc_region != NULL && cur_alloc_region == _cur_alloc_region, + "pre-condition of the call"); + assert(cur_alloc_region->is_young(), + "we only support young current alloc regions"); + + // The region is guaranteed to be young + g1_policy()->add_region_to_incremental_cset_lhs(cur_alloc_region); + _summary_bytes_used += cur_alloc_region->used(); + _cur_alloc_region = NULL; +} + +inline HeapWord* +G1CollectedHeap::attempt_allocation_locked(size_t word_size) { + assert_heap_locked_and_not_at_safepoint(); + assert(!isHumongous(word_size), "attempt_allocation_locked() " + "should not be called for humongous allocation requests"); + + // First, reread the current alloc region and retry the allocation + // in case somebody replaced it while we were waiting to get the + // Heap_lock. + HeapRegion* cur_alloc_region = _cur_alloc_region; + if (cur_alloc_region != NULL) { + HeapWord* result = allocate_from_cur_alloc_region( + cur_alloc_region, word_size, + true /* with_heap_lock */); if (result != NULL) { assert_heap_not_locked(); return result; } - assert_heap_locked(); - - // Since we couldn't successfully allocate into it, retire the - // current alloc region. + // We failed to allocate out of the current alloc region, so let's + // retire it before getting a new one. retire_cur_alloc_region(cur_alloc_region); } + assert_heap_locked(); // Try to get a new region and allocate out of it HeapWord* result = replace_cur_alloc_region_and_allocate(word_size, false, /* at_safepoint */ @@ -131,20 +186,6 @@ return NULL; } -inline void -G1CollectedHeap::retire_cur_alloc_region_common(HeapRegion* cur_alloc_region) { - assert_heap_locked_or_at_safepoint(); - assert(cur_alloc_region != NULL && cur_alloc_region == _cur_alloc_region, - "pre-condition of the call"); - assert(cur_alloc_region->is_young(), - "we only support young current alloc regions"); - - // The region is guaranteed to be young - g1_policy()->add_region_to_incremental_cset_lhs(cur_alloc_region); - _summary_bytes_used += cur_alloc_region->used(); - _cur_alloc_region = NULL; -} - // It dirties the cards that cover the block so that so that the post // write barrier never queues anything when updating objects on this // block. It is assumed (and in fact we assert) that the block diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp --- a/hotspot/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -2875,8 +2875,6 @@ // Adjust for expansion and slop. max_live_bytes = max_live_bytes + expansion_bytes; - assert(_g1->regions_accounted_for(), "Region leakage!"); - HeapRegion* hr; if (in_young_gc_mode()) { double young_start_time_sec = os::elapsedTime(); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/g1MarkSweep.cpp --- a/hotspot/src/share/vm/gc_implementation/g1/g1MarkSweep.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/g1MarkSweep.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -113,6 +113,7 @@ Threads::gc_epilogue(); CodeCache::gc_epilogue(); + JvmtiExport::gc_epilogue(); // refs processing: clean slate GenMarkSweep::_ref_processor = NULL; @@ -180,26 +181,46 @@ } class G1PrepareCompactClosure: public HeapRegionClosure { + G1CollectedHeap* _g1h; ModRefBarrierSet* _mrbs; CompactPoint _cp; + size_t _pre_used; + FreeRegionList _free_list; + HumongousRegionSet _humongous_proxy_set; void free_humongous_region(HeapRegion* hr) { - HeapWord* bot = hr->bottom(); HeapWord* end = hr->end(); assert(hr->startsHumongous(), "Only the start of a humongous region should be freed."); - G1CollectedHeap::heap()->free_region(hr); + _g1h->free_humongous_region(hr, &_pre_used, &_free_list, + &_humongous_proxy_set, false /* par */); + // Do we also need to do this for the continues humongous regions + // we just collapsed? hr->prepare_for_compaction(&_cp); // Also clear the part of the card table that will be unused after // compaction. - _mrbs->clear(MemRegion(hr->compaction_top(), hr->end())); + _mrbs->clear(MemRegion(hr->compaction_top(), end)); } public: - G1PrepareCompactClosure(CompactibleSpace* cs) : + G1PrepareCompactClosure(CompactibleSpace* cs) + : _g1h(G1CollectedHeap::heap()), + _mrbs(G1CollectedHeap::heap()->mr_bs()), _cp(NULL, cs, cs->initialize_threshold()), - _mrbs(G1CollectedHeap::heap()->mr_bs()) - {} + _pre_used(0), + _free_list("Local Free List for G1MarkSweep"), + _humongous_proxy_set("G1MarkSweep Humongous Proxy Set") { } + + void update_sets() { + // We'll recalculate total used bytes and recreate the free list + // at the end of the GC, so no point in updating those values here. + _g1h->update_sets_after_freeing_regions(0, /* pre_used */ + NULL, /* free_list */ + &_humongous_proxy_set, + false /* par */); + _free_list.remove_all(); + } + bool doHeapRegion(HeapRegion* hr) { if (hr->isHumongous()) { if (hr->startsHumongous()) { @@ -265,6 +286,7 @@ G1PrepareCompactClosure blk(sp); g1h->heap_region_iterate(&blk); + blk.update_sets(); CompactPoint perm_cp(pg, NULL, NULL); pg->prepare_for_compaction(&perm_cp); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/g1_globals.hpp --- a/hotspot/src/share/vm/gc_implementation/g1/g1_globals.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/g1_globals.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -75,21 +75,12 @@ "(0 means do not periodically generate this info); " \ "it also requires -XX:+G1SummarizeRSetStats") \ \ - diagnostic(bool, G1SummarizeZFStats, false, \ - "Summarize zero-filling info") \ - \ diagnostic(bool, G1TraceConcRefinement, false, \ "Trace G1 concurrent refinement") \ \ product(intx, G1MarkRegionStackSize, 1024 * 1024, \ "Size of the region stack for concurrent marking.") \ \ - develop(bool, G1ConcZeroFill, true, \ - "If true, run concurrent zero-filling thread") \ - \ - develop(intx, G1ConcZFMaxRegions, 1, \ - "Stop zero-filling when # of zf'd regions reaches") \ - \ develop(bool, G1SATBBarrierPrintNullPreVals, false, \ "If true, count frac of ptr writes with null pre-vals.") \ \ @@ -99,6 +90,13 @@ develop(intx, G1SATBProcessCompletedThreshold, 20, \ "Number of completed buffers that triggers log processing.") \ \ + product(uintx, G1SATBBufferEnqueueingThresholdPercent, 60, \ + "Before enqueueing them, each mutator thread tries to do some " \ + "filtering on the SATB buffers it generates. If post-filtering " \ + "the percentage of retained entries is over this threshold " \ + "the buffer will be enqueued for processing. A value of 0 " \ + "specifies that mutator threads should not do such filtering.") \ + \ develop(intx, G1ExtraRegionSurvRate, 33, \ "If the young survival rate is S, and there's room left in " \ "to-space, we will allow regions whose survival rate is up to " \ @@ -282,7 +280,20 @@ "Size of a work unit of cards claimed by a worker thread" \ "during RSet scanning.") \ \ - develop(bool, ReduceInitialCardMarksForG1, false, \ + develop(uintx, G1SecondaryFreeListAppendLength, 5, \ + "The number of regions we will add to the secondary free list " \ + "at every append operation") \ + \ + develop(bool, G1ConcRegionFreeingVerbose, false, \ + "Enables verboseness during concurrent region freeing") \ + \ + develop(bool, G1StressConcRegionFreeing, false, \ + "It stresses the concurrent region freeing operation") \ + \ + develop(uintx, G1StressConcRegionFreeingDelayMillis, 0, \ + "Artificial delay during concurrent region freeing") \ + \ + develop(bool, ReduceInitialCardMarksForG1, false, \ "When ReduceInitialCardMarks is true, this flag setting " \ " controls whether G1 allows the RICM optimization") diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/heapRegion.cpp --- a/hotspot/src/share/vm/gc_implementation/g1/heapRegion.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/heapRegion.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,7 +23,6 @@ */ #include "precompiled.hpp" -#include "gc_implementation/g1/concurrentZFThread.hpp" #include "gc_implementation/g1/g1BlockOffsetTable.inline.hpp" #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" #include "gc_implementation/g1/g1OopClosures.inline.hpp" @@ -348,22 +347,20 @@ } void HeapRegion::hr_clear(bool par, bool clear_space) { - _humongous_type = NotHumongous; - _humongous_start_region = NULL; + assert(_humongous_type == NotHumongous, + "we should have already filtered out humongous regions"); + assert(_humongous_start_region == NULL, + "we should have already filtered out humongous regions"); + assert(_end == _orig_end, + "we should have already filtered out humongous regions"); + _in_collection_set = false; _is_gc_alloc_region = false; - // Age stuff (if parallel, this will be done separately, since it needs - // to be sequential). - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - set_young_index_in_cset(-1); uninstall_surv_rate_group(); set_young_type(NotYoung); - // In case it had been the start of a humongous sequence, reset its end. - set_end(_orig_end); - if (!par) { // If this is parallel, this will be done later. HeapRegionRemSet* hrrs = rem_set(); @@ -386,26 +383,49 @@ } // -void HeapRegion::set_startsHumongous(HeapWord* new_end) { +void HeapRegion::set_startsHumongous(HeapWord* new_top, HeapWord* new_end) { + assert(!isHumongous(), "sanity / pre-condition"); assert(end() == _orig_end, "Should be normal before the humongous object allocation"); assert(top() == bottom(), "should be empty"); + assert(bottom() <= new_top && new_top <= new_end, "pre-condition"); _humongous_type = StartsHumongous; _humongous_start_region = this; set_end(new_end); - _offsets.set_for_starts_humongous(new_end); + _offsets.set_for_starts_humongous(new_top); } -void HeapRegion::set_continuesHumongous(HeapRegion* start) { +void HeapRegion::set_continuesHumongous(HeapRegion* first_hr) { + assert(!isHumongous(), "sanity / pre-condition"); assert(end() == _orig_end, "Should be normal before the humongous object allocation"); assert(top() == bottom(), "should be empty"); - assert(start->startsHumongous(), "pre-condition"); + assert(first_hr->startsHumongous(), "pre-condition"); _humongous_type = ContinuesHumongous; - _humongous_start_region = start; + _humongous_start_region = first_hr; +} + +void HeapRegion::set_notHumongous() { + assert(isHumongous(), "pre-condition"); + + if (startsHumongous()) { + assert(top() <= end(), "pre-condition"); + set_end(_orig_end); + if (top() > end()) { + // at least one "continues humongous" region after it + set_top(end()); + } + } else { + // continues humongous + assert(end() == _orig_end, "sanity"); + } + + assert(capacity() == (size_t) HeapRegion::GrainBytes, "pre-condition"); + _humongous_type = NotHumongous; + _humongous_start_region = NULL; } bool HeapRegion::claimHeapRegion(jint claimValue) { @@ -442,15 +462,6 @@ return low; } -void HeapRegion::set_next_on_unclean_list(HeapRegion* r) { - assert(r == NULL || r->is_on_unclean_list(), "Malformed unclean list."); - _next_in_special_set = r; -} - -void HeapRegion::set_on_unclean_list(bool b) { - _is_on_unclean_list = b; -} - void HeapRegion::initialize(MemRegion mr, bool clear_space, bool mangle_space) { G1OffsetTableContigSpace::initialize(mr, false, mangle_space); hr_clear(false/*par*/, clear_space); @@ -468,15 +479,16 @@ _hrs_index(-1), _humongous_type(NotHumongous), _humongous_start_region(NULL), _in_collection_set(false), _is_gc_alloc_region(false), - _is_on_free_list(false), _is_on_unclean_list(false), _next_in_special_set(NULL), _orig_end(NULL), _claimed(InitialClaimValue), _evacuation_failed(false), _prev_marked_bytes(0), _next_marked_bytes(0), _sort_index(-1), _young_type(NotYoung), _next_young_region(NULL), - _next_dirty_cards_region(NULL), - _young_index_in_cset(-1), _surv_rate_group(NULL), _age_index(-1), - _rem_set(NULL), _zfs(NotZeroFilled), - _recorded_rs_length(0), _predicted_elapsed_time_ms(0), + _next_dirty_cards_region(NULL), _next(NULL), _pending_removal(false), +#ifdef ASSERT + _containing_set(NULL), +#endif // ASSERT + _young_index_in_cset(-1), _surv_rate_group(NULL), _age_index(-1), + _rem_set(NULL), _recorded_rs_length(0), _predicted_elapsed_time_ms(0), _predicted_bytes_to_copy(0) { _orig_end = mr.end(); @@ -551,86 +563,6 @@ oops_in_mr_iterate(MemRegion(bottom(), saved_mark_word()), cl); } -#ifdef DEBUG -HeapWord* HeapRegion::allocate(size_t size) { - jint state = zero_fill_state(); - assert(!G1CollectedHeap::heap()->allocs_are_zero_filled() || - zero_fill_is_allocated(), - "When ZF is on, only alloc in ZF'd regions"); - return G1OffsetTableContigSpace::allocate(size); -} -#endif - -void HeapRegion::set_zero_fill_state_work(ZeroFillState zfs) { - assert(ZF_mon->owned_by_self() || - Universe::heap()->is_gc_active(), - "Must hold the lock or be a full GC to modify."); -#ifdef ASSERT - if (top() != bottom() && zfs != Allocated) { - ResourceMark rm; - stringStream region_str; - print_on(®ion_str); - assert(top() == bottom() || zfs == Allocated, - err_msg("Region must be empty, or we must be setting it to allocated. " - "_zfs=%d, zfs=%d, region: %s", _zfs, zfs, region_str.as_string())); - } -#endif - _zfs = zfs; -} - -void HeapRegion::set_zero_fill_complete() { - set_zero_fill_state_work(ZeroFilled); - if (ZF_mon->owned_by_self()) { - ZF_mon->notify_all(); - } -} - - -void HeapRegion::ensure_zero_filled() { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - ensure_zero_filled_locked(); -} - -void HeapRegion::ensure_zero_filled_locked() { - assert(ZF_mon->owned_by_self(), "Precondition"); - bool should_ignore_zf = SafepointSynchronize::is_at_safepoint(); - assert(should_ignore_zf || Heap_lock->is_locked(), - "Either we're in a GC or we're allocating a region."); - switch (zero_fill_state()) { - case HeapRegion::NotZeroFilled: - set_zero_fill_in_progress(Thread::current()); - { - ZF_mon->unlock(); - Copy::fill_to_words(bottom(), capacity()/HeapWordSize); - ZF_mon->lock_without_safepoint_check(); - } - // A trap. - guarantee(zero_fill_state() == HeapRegion::ZeroFilling - && zero_filler() == Thread::current(), - "AHA! Tell Dave D if you see this..."); - set_zero_fill_complete(); - // gclog_or_tty->print_cr("Did sync ZF."); - ConcurrentZFThread::note_sync_zfs(); - break; - case HeapRegion::ZeroFilling: - if (should_ignore_zf) { - // We can "break" the lock and take over the work. - Copy::fill_to_words(bottom(), capacity()/HeapWordSize); - set_zero_fill_complete(); - ConcurrentZFThread::note_sync_zfs(); - break; - } else { - ConcurrentZFThread::wait_for_ZF_completed(this); - } - case HeapRegion::ZeroFilled: - // Nothing to do. - break; - case HeapRegion::Allocated: - guarantee(false, "Should not call on allocated regions."); - } - assert(zero_fill_state() == HeapRegion::ZeroFilled, "Post"); -} - HeapWord* HeapRegion::object_iterate_mem_careful(MemRegion mr, ObjectClosure* cl) { @@ -782,9 +714,6 @@ verify(allow_dirty, /* use_prev_marking */ true, /* failures */ &dummy); } -#define OBJ_SAMPLE_INTERVAL 0 -#define BLOCK_SAMPLE_INTERVAL 100 - // This really ought to be commoned up into OffsetTableContigSpace somehow. // We would need a mechanism to make that code skip dead objects. @@ -795,83 +724,125 @@ *failures = false; HeapWord* p = bottom(); HeapWord* prev_p = NULL; - int objs = 0; - int blocks = 0; VerifyLiveClosure vl_cl(g1, use_prev_marking); bool is_humongous = isHumongous(); + bool do_bot_verify = !is_young(); size_t object_num = 0; while (p < top()) { - size_t size = oop(p)->size(); - if (is_humongous != g1->isHumongous(size)) { + oop obj = oop(p); + size_t obj_size = obj->size(); + object_num += 1; + + if (is_humongous != g1->isHumongous(obj_size)) { gclog_or_tty->print_cr("obj "PTR_FORMAT" is of %shumongous size (" SIZE_FORMAT" words) in a %shumongous region", - p, g1->isHumongous(size) ? "" : "non-", - size, is_humongous ? "" : "non-"); + p, g1->isHumongous(obj_size) ? "" : "non-", + obj_size, is_humongous ? "" : "non-"); *failures = true; + return; + } + + // If it returns false, verify_for_object() will output the + // appropriate messasge. + if (do_bot_verify && !_offsets.verify_for_object(p, obj_size)) { + *failures = true; + return; } - object_num += 1; - if (blocks == BLOCK_SAMPLE_INTERVAL) { - HeapWord* res = block_start_const(p + (size/2)); - if (p != res) { - gclog_or_tty->print_cr("offset computation 1 for "PTR_FORMAT" and " - SIZE_FORMAT" returned "PTR_FORMAT, - p, size, res); + + if (!g1->is_obj_dead_cond(obj, this, use_prev_marking)) { + if (obj->is_oop()) { + klassOop klass = obj->klass(); + if (!klass->is_perm()) { + gclog_or_tty->print_cr("klass "PTR_FORMAT" of object "PTR_FORMAT" " + "not in perm", klass, obj); + *failures = true; + return; + } else if (!klass->is_klass()) { + gclog_or_tty->print_cr("klass "PTR_FORMAT" of object "PTR_FORMAT" " + "not a klass", klass, obj); + *failures = true; + return; + } else { + vl_cl.set_containing_obj(obj); + obj->oop_iterate(&vl_cl); + if (vl_cl.failures()) { + *failures = true; + } + if (G1MaxVerifyFailures >= 0 && + vl_cl.n_failures() >= G1MaxVerifyFailures) { + return; + } + } + } else { + gclog_or_tty->print_cr(PTR_FORMAT" no an oop", obj); *failures = true; return; } - blocks = 0; - } else { - blocks++; - } - if (objs == OBJ_SAMPLE_INTERVAL) { - oop obj = oop(p); - if (!g1->is_obj_dead_cond(obj, this, use_prev_marking)) { - if (obj->is_oop()) { - klassOop klass = obj->klass(); - if (!klass->is_perm()) { - gclog_or_tty->print_cr("klass "PTR_FORMAT" of object "PTR_FORMAT" " - "not in perm", klass, obj); - *failures = true; - return; - } else if (!klass->is_klass()) { - gclog_or_tty->print_cr("klass "PTR_FORMAT" of object "PTR_FORMAT" " - "not a klass", klass, obj); - *failures = true; - return; - } else { - vl_cl.set_containing_obj(obj); - obj->oop_iterate(&vl_cl); - if (vl_cl.failures()) { - *failures = true; - } - if (G1MaxVerifyFailures >= 0 && - vl_cl.n_failures() >= G1MaxVerifyFailures) { - return; - } - } - } else { - gclog_or_tty->print_cr(PTR_FORMAT" no an oop", obj); - *failures = true; - return; - } - } - objs = 0; - } else { - objs++; } prev_p = p; - p += size; + p += obj_size; + } + + if (p != top()) { + gclog_or_tty->print_cr("end of last object "PTR_FORMAT" " + "does not match top "PTR_FORMAT, p, top()); + *failures = true; + return; } - HeapWord* rend = end(); - HeapWord* rtop = top(); - if (rtop < rend) { - HeapWord* res = block_start_const(rtop + (rend - rtop) / 2); - if (res != rtop) { - gclog_or_tty->print_cr("offset computation 2 for "PTR_FORMAT" and " - PTR_FORMAT" returned "PTR_FORMAT, - rtop, rend, res); + + HeapWord* the_end = end(); + assert(p == top(), "it should still hold"); + // Do some extra BOT consistency checking for addresses in the + // range [top, end). BOT look-ups in this range should yield + // top. No point in doing that if top == end (there's nothing there). + if (p < the_end) { + // Look up top + HeapWord* addr_1 = p; + HeapWord* b_start_1 = _offsets.block_start_const(addr_1); + if (b_start_1 != p) { + gclog_or_tty->print_cr("BOT look up for top: "PTR_FORMAT" " + " yielded "PTR_FORMAT", expecting "PTR_FORMAT, + addr_1, b_start_1, p); + *failures = true; + return; + } + + // Look up top + 1 + HeapWord* addr_2 = p + 1; + if (addr_2 < the_end) { + HeapWord* b_start_2 = _offsets.block_start_const(addr_2); + if (b_start_2 != p) { + gclog_or_tty->print_cr("BOT look up for top + 1: "PTR_FORMAT" " + " yielded "PTR_FORMAT", expecting "PTR_FORMAT, + addr_2, b_start_2, p); *failures = true; return; + } + } + + // Look up an address between top and end + size_t diff = pointer_delta(the_end, p) / 2; + HeapWord* addr_3 = p + diff; + if (addr_3 < the_end) { + HeapWord* b_start_3 = _offsets.block_start_const(addr_3); + if (b_start_3 != p) { + gclog_or_tty->print_cr("BOT look up for top + diff: "PTR_FORMAT" " + " yielded "PTR_FORMAT", expecting "PTR_FORMAT, + addr_3, b_start_3, p); + *failures = true; + return; + } + } + + // Loook up end - 1 + HeapWord* addr_4 = the_end - 1; + HeapWord* b_start_4 = _offsets.block_start_const(addr_4); + if (b_start_4 != p) { + gclog_or_tty->print_cr("BOT look up for end - 1: "PTR_FORMAT" " + " yielded "PTR_FORMAT", expecting "PTR_FORMAT, + addr_4, b_start_4, p); + *failures = true; + return; } } @@ -880,12 +851,6 @@ "but has "SIZE_FORMAT", objects", bottom(), end(), object_num); *failures = true; - } - - if (p != top()) { - gclog_or_tty->print_cr("end of last object "PTR_FORMAT" " - "does not match top "PTR_FORMAT, p, top()); - *failures = true; return; } } @@ -976,67 +941,3 @@ _offsets.set_space(this); initialize(mr, !is_zeroed, SpaceDecorator::Mangle); } - -size_t RegionList::length() { - size_t len = 0; - HeapRegion* cur = hd(); - DEBUG_ONLY(HeapRegion* last = NULL); - while (cur != NULL) { - len++; - DEBUG_ONLY(last = cur); - cur = get_next(cur); - } - assert(last == tl(), "Invariant"); - return len; -} - -void RegionList::insert_before_head(HeapRegion* r) { - assert(well_formed(), "Inv"); - set_next(r, hd()); - _hd = r; - _sz++; - if (tl() == NULL) _tl = r; - assert(well_formed(), "Inv"); -} - -void RegionList::prepend_list(RegionList* new_list) { - assert(well_formed(), "Precondition"); - assert(new_list->well_formed(), "Precondition"); - HeapRegion* new_tl = new_list->tl(); - if (new_tl != NULL) { - set_next(new_tl, hd()); - _hd = new_list->hd(); - _sz += new_list->sz(); - if (tl() == NULL) _tl = new_list->tl(); - } else { - assert(new_list->hd() == NULL && new_list->sz() == 0, "Inv"); - } - assert(well_formed(), "Inv"); -} - -void RegionList::delete_after(HeapRegion* r) { - assert(well_formed(), "Precondition"); - HeapRegion* next = get_next(r); - assert(r != NULL, "Precondition"); - HeapRegion* next_tl = get_next(next); - set_next(r, next_tl); - dec_sz(); - if (next == tl()) { - assert(next_tl == NULL, "Inv"); - _tl = r; - } - assert(well_formed(), "Inv"); -} - -HeapRegion* RegionList::pop() { - assert(well_formed(), "Inv"); - HeapRegion* res = hd(); - if (res != NULL) { - _hd = get_next(res); - _sz--; - set_next(res, NULL); - if (sz() == 0) _tl = NULL; - } - assert(well_formed(), "Inv"); - return res; -} diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/heapRegion.hpp --- a/hotspot/src/share/vm/gc_implementation/g1/heapRegion.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/heapRegion.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -50,6 +50,11 @@ class HeapRegionRemSet; class HeapRegionRemSetIterator; class HeapRegion; +class HeapRegionSetBase; + +#define HR_FORMAT "%d:["PTR_FORMAT","PTR_FORMAT","PTR_FORMAT"]" +#define HR_FORMAT_PARAMS(__hr) (__hr)->hrs_index(), (__hr)->bottom(), \ + (__hr)->top(), (__hr)->end() // A dirty card to oop closure for heap regions. It // knows how to get the G1 heap and how to use the bitmap @@ -173,6 +178,19 @@ virtual HeapWord* cross_threshold(HeapWord* start, HeapWord* end); virtual void print() const; + + void reset_bot() { + _offsets.zero_bottom_entry(); + _offsets.initialize_threshold(); + } + + void update_bot_for_object(HeapWord* start, size_t word_size) { + _offsets.alloc_block(start, word_size); + } + + void print_bot_on(outputStream* out) { + _offsets.print_on(out); + } }; class HeapRegion: public G1OffsetTableContigSpace { @@ -214,12 +232,6 @@ // True iff the region is in current collection_set. bool _in_collection_set; - // True iff the region is on the unclean list, waiting to be zero filled. - bool _is_on_unclean_list; - - // True iff the region is on the free list, ready for allocation. - bool _is_on_free_list; - // Is this or has it been an allocation region in the current collection // pause. bool _is_gc_alloc_region; @@ -241,6 +253,13 @@ // Next region whose cards need cleaning HeapRegion* _next_dirty_cards_region; + // Fields used by the HeapRegionSetBase class and subclasses. + HeapRegion* _next; +#ifdef ASSERT + HeapRegionSetBase* _containing_set; +#endif // ASSERT + bool _pending_removal; + // For parallel heapRegion traversal. jint _claimed; @@ -292,10 +311,6 @@ _top_at_conc_mark_count = bot; } - jint _zfs; // A member of ZeroFillState. Protected by ZF_lock. - Thread* _zero_filler; // If _zfs is ZeroFilling, the thread that (last) - // made it so. - void set_young_type(YoungType new_type) { //assert(_young_type != new_type, "setting the same type" ); // TODO: add more assertions here @@ -349,15 +364,14 @@ RebuildRSClaimValue = 5 }; - // Concurrent refinement requires contiguous heap regions (in which TLABs - // might be allocated) to be zero-filled. Each region therefore has a - // zero-fill-state. - enum ZeroFillState { - NotZeroFilled, - ZeroFilling, - ZeroFilled, - Allocated - }; + inline HeapWord* par_allocate_no_bot_updates(size_t word_size) { + assert(is_young(), "we can only skip BOT updates on young regions"); + return ContiguousSpace::par_allocate(word_size); + } + inline HeapWord* allocate_no_bot_updates(size_t word_size) { + assert(is_young(), "we can only skip BOT updates on young regions"); + return ContiguousSpace::allocate(word_size); + } // If this region is a member of a HeapRegionSeq, the index in that // sequence, otherwise -1. @@ -404,13 +418,38 @@ return _humongous_start_region; } - // Causes the current region to represent a humongous object spanning "n" - // regions. - void set_startsHumongous(HeapWord* new_end); + // Makes the current region be a "starts humongous" region, i.e., + // the first region in a series of one or more contiguous regions + // that will contain a single "humongous" object. The two parameters + // are as follows: + // + // new_top : The new value of the top field of this region which + // points to the end of the humongous object that's being + // allocated. If there is more than one region in the series, top + // will lie beyond this region's original end field and on the last + // region in the series. + // + // new_end : The new value of the end field of this region which + // points to the end of the last region in the series. If there is + // one region in the series (namely: this one) end will be the same + // as the original end of this region. + // + // Updating top and end as described above makes this region look as + // if it spans the entire space taken up by all the regions in the + // series and an single allocation moved its top to new_top. This + // ensures that the space (capacity / allocated) taken up by all + // humongous regions can be calculated by just looking at the + // "starts humongous" regions and by ignoring the "continues + // humongous" regions. + void set_startsHumongous(HeapWord* new_top, HeapWord* new_end); - // The regions that continue a humongous sequence should be added using - // this method, in increasing address order. - void set_continuesHumongous(HeapRegion* start); + // Makes the current region be a "continues humongous' + // region. first_hr is the "start humongous" region of the series + // which this region will be part of. + void set_continuesHumongous(HeapRegion* first_hr); + + // Unsets the humongous-related fields on the region. + void set_notHumongous(); // If the region has a remembered set, return a pointer to it. HeapRegionRemSet* rem_set() const { @@ -458,45 +497,56 @@ _next_in_special_set = r; } - bool is_on_free_list() { - return _is_on_free_list; - } + // Methods used by the HeapRegionSetBase class and subclasses. - void set_on_free_list(bool b) { - _is_on_free_list = b; - } + // Getter and setter for the next field used to link regions into + // linked lists. + HeapRegion* next() { return _next; } + + void set_next(HeapRegion* next) { _next = next; } - HeapRegion* next_from_free_list() { - assert(is_on_free_list(), - "Should only invoke on free space."); - assert(_next_in_special_set == NULL || - _next_in_special_set->is_on_free_list(), - "Malformed Free List."); - return _next_in_special_set; - } + // Every region added to a set is tagged with a reference to that + // set. This is used for doing consistency checking to make sure that + // the contents of a set are as they should be and it's only + // available in non-product builds. +#ifdef ASSERT + void set_containing_set(HeapRegionSetBase* containing_set) { + assert((containing_set == NULL && _containing_set != NULL) || + (containing_set != NULL && _containing_set == NULL), + err_msg("containing_set: "PTR_FORMAT" " + "_containing_set: "PTR_FORMAT, + containing_set, _containing_set)); + + _containing_set = containing_set; +} - void set_next_on_free_list(HeapRegion* r) { - assert(r == NULL || r->is_on_free_list(), "Malformed free list."); - _next_in_special_set = r; - } + HeapRegionSetBase* containing_set() { return _containing_set; } +#else // ASSERT + void set_containing_set(HeapRegionSetBase* containing_set) { } - bool is_on_unclean_list() { - return _is_on_unclean_list; - } + // containing_set() is only used in asserts so there's not reason + // to provide a dummy version of it. +#endif // ASSERT - void set_on_unclean_list(bool b); + // If we want to remove regions from a list in bulk we can simply tag + // them with the pending_removal tag and call the + // remove_all_pending() method on the list. - HeapRegion* next_from_unclean_list() { - assert(is_on_unclean_list(), - "Should only invoke on unclean space."); - assert(_next_in_special_set == NULL || - _next_in_special_set->is_on_unclean_list(), - "Malformed unclean List."); - return _next_in_special_set; + bool pending_removal() { return _pending_removal; } + + void set_pending_removal(bool pending_removal) { + // We can only set pending_removal to true, if it's false and the + // region belongs to a set. + assert(!pending_removal || + (!_pending_removal && containing_set() != NULL), "pre-condition"); + // We can only set pending_removal to false, if it's true and the + // region does not belong to a set. + assert( pending_removal || + ( _pending_removal && containing_set() == NULL), "pre-condition"); + + _pending_removal = pending_removal; } - void set_next_on_unclean_list(HeapRegion* r); - HeapRegion* get_next_young_region() { return _next_young_region; } void set_next_young_region(HeapRegion* hr) { _next_young_region = hr; @@ -515,11 +565,6 @@ void initialize(MemRegion mr, bool clear_space, bool mangle_space); - // Ensure that "this" is zero-filled. - void ensure_zero_filled(); - // This one requires that the calling thread holds ZF_mon. - void ensure_zero_filled_locked(); - // Get the start of the unmarked area in this region. HeapWord* prev_top_at_mark_start() const { return _prev_top_at_mark_start; } HeapWord* next_top_at_mark_start() const { return _next_top_at_mark_start; } @@ -754,36 +799,6 @@ // "end" of the region if there is no such block. HeapWord* next_block_start_careful(HeapWord* addr); - // Returns the zero-fill-state of the current region. - ZeroFillState zero_fill_state() { return (ZeroFillState)_zfs; } - bool zero_fill_is_allocated() { return _zfs == Allocated; } - Thread* zero_filler() { return _zero_filler; } - - // Indicate that the contents of the region are unknown, and therefore - // might require zero-filling. - void set_zero_fill_needed() { - set_zero_fill_state_work(NotZeroFilled); - } - void set_zero_fill_in_progress(Thread* t) { - set_zero_fill_state_work(ZeroFilling); - _zero_filler = t; - } - void set_zero_fill_complete(); - void set_zero_fill_allocated() { - set_zero_fill_state_work(Allocated); - } - - void set_zero_fill_state_work(ZeroFillState zfs); - - // This is called when a full collection shrinks the heap. - // We want to set the heap region to a value which says - // it is no longer part of the heap. For now, we'll let "NotZF" fill - // that role. - void reset_zero_fill() { - set_zero_fill_state_work(NotZeroFilled); - _zero_filler = NULL; - } - size_t recorded_rs_length() const { return _recorded_rs_length; } double predicted_elapsed_time_ms() const { return _predicted_elapsed_time_ms; } size_t predicted_bytes_to_copy() const { return _predicted_bytes_to_copy; } @@ -822,10 +837,6 @@ // Override; it uses the "prev" marking information virtual void verify(bool allow_dirty) const; - -#ifdef DEBUG - HeapWord* allocate(size_t size); -#endif }; // HeapRegionClosure is used for iterating over regions. @@ -848,113 +859,6 @@ bool complete() { return _complete; } }; -// A linked lists of heap regions. It leaves the "next" field -// unspecified; that's up to subtypes. -class RegionList VALUE_OBJ_CLASS_SPEC { -protected: - virtual HeapRegion* get_next(HeapRegion* chr) = 0; - virtual void set_next(HeapRegion* chr, - HeapRegion* new_next) = 0; - - HeapRegion* _hd; - HeapRegion* _tl; - size_t _sz; - - // Protected constructor because this type is only meaningful - // when the _get/_set next functions are defined. - RegionList() : _hd(NULL), _tl(NULL), _sz(0) {} -public: - void reset() { - _hd = NULL; - _tl = NULL; - _sz = 0; - } - HeapRegion* hd() { return _hd; } - HeapRegion* tl() { return _tl; } - size_t sz() { return _sz; } - size_t length(); - - bool well_formed() { - return - ((hd() == NULL && tl() == NULL && sz() == 0) - || (hd() != NULL && tl() != NULL && sz() > 0)) - && (sz() == length()); - } - virtual void insert_before_head(HeapRegion* r); - void prepend_list(RegionList* new_list); - virtual HeapRegion* pop(); - void dec_sz() { _sz--; } - // Requires that "r" is an element of the list, and is not the tail. - void delete_after(HeapRegion* r); -}; - -class EmptyNonHRegionList: public RegionList { -protected: - // Protected constructor because this type is only meaningful - // when the _get/_set next functions are defined. - EmptyNonHRegionList() : RegionList() {} - -public: - void insert_before_head(HeapRegion* r) { - // assert(r->is_empty(), "Better be empty"); - assert(!r->isHumongous(), "Better not be humongous."); - RegionList::insert_before_head(r); - } - void prepend_list(EmptyNonHRegionList* new_list) { - // assert(new_list->hd() == NULL || new_list->hd()->is_empty(), - // "Better be empty"); - assert(new_list->hd() == NULL || !new_list->hd()->isHumongous(), - "Better not be humongous."); - // assert(new_list->tl() == NULL || new_list->tl()->is_empty(), - // "Better be empty"); - assert(new_list->tl() == NULL || !new_list->tl()->isHumongous(), - "Better not be humongous."); - RegionList::prepend_list(new_list); - } -}; - -class UncleanRegionList: public EmptyNonHRegionList { -public: - HeapRegion* get_next(HeapRegion* hr) { - return hr->next_from_unclean_list(); - } - void set_next(HeapRegion* hr, HeapRegion* new_next) { - hr->set_next_on_unclean_list(new_next); - } - - UncleanRegionList() : EmptyNonHRegionList() {} - - void insert_before_head(HeapRegion* r) { - assert(!r->is_on_free_list(), - "Better not already be on free list"); - assert(!r->is_on_unclean_list(), - "Better not already be on unclean list"); - r->set_zero_fill_needed(); - r->set_on_unclean_list(true); - EmptyNonHRegionList::insert_before_head(r); - } - void prepend_list(UncleanRegionList* new_list) { - assert(new_list->tl() == NULL || !new_list->tl()->is_on_free_list(), - "Better not already be on free list"); - assert(new_list->tl() == NULL || new_list->tl()->is_on_unclean_list(), - "Better already be marked as on unclean list"); - assert(new_list->hd() == NULL || !new_list->hd()->is_on_free_list(), - "Better not already be on free list"); - assert(new_list->hd() == NULL || new_list->hd()->is_on_unclean_list(), - "Better already be marked as on unclean list"); - EmptyNonHRegionList::prepend_list(new_list); - } - HeapRegion* pop() { - HeapRegion* res = RegionList::pop(); - if (res != NULL) res->set_on_unclean_list(false); - return res; - } -}; - -// Local Variables: *** -// c-indentation-style: gnu *** -// End: *** - #endif // SERIALGC #endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGION_HPP diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/heapRegionSeq.cpp --- a/hotspot/src/share/vm/gc_implementation/g1/heapRegionSeq.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/heapRegionSeq.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -65,152 +65,6 @@ // Private methods. -HeapWord* -HeapRegionSeq::alloc_obj_from_region_index(int ind, size_t word_size) { - assert(G1CollectedHeap::isHumongous(word_size), - "Allocation size should be humongous"); - int cur = ind; - int first = cur; - size_t sumSizes = 0; - while (cur < _regions.length() && sumSizes < word_size) { - // Loop invariant: - // For all i in [first, cur): - // _regions.at(i)->is_empty() - // && _regions.at(i) is contiguous with its predecessor, if any - // && sumSizes is the sum of the sizes of the regions in the interval - // [first, cur) - HeapRegion* curhr = _regions.at(cur); - if (curhr->is_empty() - && (first == cur - || (_regions.at(cur-1)->end() == - curhr->bottom()))) { - sumSizes += curhr->capacity() / HeapWordSize; - } else { - first = cur + 1; - sumSizes = 0; - } - cur++; - } - if (sumSizes >= word_size) { - _alloc_search_start = cur; - - // We need to initialize the region(s) we just discovered. This is - // a bit tricky given that it can happen concurrently with - // refinement threads refining cards on these regions and - // potentially wanting to refine the BOT as they are scanning - // those cards (this can happen shortly after a cleanup; see CR - // 6991377). So we have to set up the region(s) carefully and in - // a specific order. - - // Currently, allocs_are_zero_filled() returns false. The zero - // filling infrastructure will be going away soon (see CR 6977804). - // So no need to do anything else here. - bool zf = G1CollectedHeap::heap()->allocs_are_zero_filled(); - assert(!zf, "not supported"); - - // This will be the "starts humongous" region. - HeapRegion* first_hr = _regions.at(first); - { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - first_hr->set_zero_fill_allocated(); - } - // The header of the new object will be placed at the bottom of - // the first region. - HeapWord* new_obj = first_hr->bottom(); - // This will be the new end of the first region in the series that - // should also match the end of the last region in the seriers. - // (Note: sumSizes = "region size" x "number of regions we found"). - HeapWord* new_end = new_obj + sumSizes; - // This will be the new top of the first region that will reflect - // this allocation. - HeapWord* new_top = new_obj + word_size; - - // First, we need to zero the header of the space that we will be - // allocating. When we update top further down, some refinement - // threads might try to scan the region. By zeroing the header we - // ensure that any thread that will try to scan the region will - // come across the zero klass word and bail out. - // - // NOTE: It would not have been correct to have used - // CollectedHeap::fill_with_object() and make the space look like - // an int array. The thread that is doing the allocation will - // later update the object header to a potentially different array - // type and, for a very short period of time, the klass and length - // fields will be inconsistent. This could cause a refinement - // thread to calculate the object size incorrectly. - Copy::fill_to_words(new_obj, oopDesc::header_size(), 0); - - // We will set up the first region as "starts humongous". This - // will also update the BOT covering all the regions to reflect - // that there is a single object that starts at the bottom of the - // first region. - first_hr->set_startsHumongous(new_end); - - // Then, if there are any, we will set up the "continues - // humongous" regions. - HeapRegion* hr = NULL; - for (int i = first + 1; i < cur; ++i) { - hr = _regions.at(i); - { - MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); - hr->set_zero_fill_allocated(); - } - hr->set_continuesHumongous(first_hr); - } - // If we have "continues humongous" regions (hr != NULL), then the - // end of the last one should match new_end. - assert(hr == NULL || hr->end() == new_end, "sanity"); - - // Up to this point no concurrent thread would have been able to - // do any scanning on any region in this series. All the top - // fields still point to bottom, so the intersection between - // [bottom,top] and [card_start,card_end] will be empty. Before we - // update the top fields, we'll do a storestore to make sure that - // no thread sees the update to top before the zeroing of the - // object header and the BOT initialization. - OrderAccess::storestore(); - - // Now that the BOT and the object header have been initialized, - // we can update top of the "starts humongous" region. - assert(first_hr->bottom() < new_top && new_top <= first_hr->end(), - "new_top should be in this region"); - first_hr->set_top(new_top); - - // Now, we will update the top fields of the "continues humongous" - // regions. The reason we need to do this is that, otherwise, - // these regions would look empty and this will confuse parts of - // G1. For example, the code that looks for a consecutive number - // of empty regions will consider them empty and try to - // re-allocate them. We can extend is_empty() to also include - // !continuesHumongous(), but it is easier to just update the top - // fields here. - hr = NULL; - for (int i = first + 1; i < cur; ++i) { - hr = _regions.at(i); - if ((i + 1) == cur) { - // last continues humongous region - assert(hr->bottom() < new_top && new_top <= hr->end(), - "new_top should fall on this region"); - hr->set_top(new_top); - } else { - // not last one - assert(new_top > hr->end(), "new_top should be above this region"); - hr->set_top(hr->end()); - } - } - // If we have continues humongous regions (hr != NULL), then the - // end of the last one should match new_end and its top should - // match new_top. - assert(hr == NULL || - (hr->end() == new_end && hr->top() == new_top), "sanity"); - - return new_obj; - } else { - // If we started from the beginning, we want to know why we can't alloc. - return NULL; - } -} - void HeapRegionSeq::print_empty_runs() { int empty_run = 0; int n_empty = 0; @@ -284,13 +138,67 @@ return res; } -HeapWord* HeapRegionSeq::obj_allocate(size_t word_size) { - int cur = _alloc_search_start; - // Make sure "cur" is a valid index. - assert(cur >= 0, "Invariant."); - HeapWord* res = alloc_obj_from_region_index(cur, word_size); - if (res == NULL) - res = alloc_obj_from_region_index(0, word_size); +int HeapRegionSeq::find_contiguous_from(int from, size_t num) { + assert(num > 1, "pre-condition"); + assert(0 <= from && from <= _regions.length(), + err_msg("from: %d should be valid and <= than %d", + from, _regions.length())); + + int curr = from; + int first = -1; + size_t num_so_far = 0; + while (curr < _regions.length() && num_so_far < num) { + HeapRegion* curr_hr = _regions.at(curr); + if (curr_hr->is_empty()) { + if (first == -1) { + first = curr; + num_so_far = 1; + } else { + num_so_far += 1; + } + } else { + first = -1; + num_so_far = 0; + } + curr += 1; + } + + assert(num_so_far <= num, "post-condition"); + if (num_so_far == num) { + // we find enough space for the humongous object + assert(from <= first && first < _regions.length(), "post-condition"); + assert(first < curr && (curr - first) == (int) num, "post-condition"); + for (int i = first; i < first + (int) num; ++i) { + assert(_regions.at(i)->is_empty(), "post-condition"); + } + return first; + } else { + // we failed to find enough space for the humongous object + return -1; + } +} + +int HeapRegionSeq::find_contiguous(size_t num) { + assert(num > 1, "otherwise we should not be calling this"); + assert(0 <= _alloc_search_start && _alloc_search_start <= _regions.length(), + err_msg("_alloc_search_start: %d should be valid and <= than %d", + _alloc_search_start, _regions.length())); + + int start = _alloc_search_start; + int res = find_contiguous_from(start, num); + if (res == -1 && start != 0) { + // Try starting from the beginning. If _alloc_search_start was 0, + // no point in doing this again. + res = find_contiguous_from(0, num); + } + if (res != -1) { + assert(0 <= res && res < _regions.length(), + err_msg("res: %d should be valid", res)); + _alloc_search_start = res + (int) num; + } + assert(0 < _alloc_search_start && _alloc_search_start <= _regions.length(), + err_msg("_alloc_search_start: %d should be valid", + _alloc_search_start)); return res; } @@ -376,6 +284,10 @@ MemRegion HeapRegionSeq::shrink_by(size_t shrink_bytes, size_t& num_regions_deleted) { + // Reset this in case it's currently pointing into the regions that + // we just removed. + _alloc_search_start = 0; + assert(shrink_bytes % os::vm_page_size() == 0, "unaligned"); assert(shrink_bytes % HeapRegion::GrainBytes == 0, "unaligned"); @@ -395,7 +307,6 @@ } assert(cur == _regions.top(), "Should be top"); if (!cur->is_empty()) break; - cur->reset_zero_fill(); shrink_bytes -= cur->capacity(); num_regions_deleted++; _regions.pop(); @@ -410,7 +321,6 @@ return MemRegion(last_start, end); } - class PrintHeapRegionClosure : public HeapRegionClosure { public: bool doHeapRegion(HeapRegion* r) { diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/heapRegionSeq.hpp --- a/hotspot/src/share/vm/gc_implementation/g1/heapRegionSeq.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/heapRegionSeq.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -41,9 +41,9 @@ // (For efficiency only; private to obj_allocate after initialization.) int _alloc_search_start; - // Attempts to allocate a block of the (assumed humongous) word_size, - // starting at the region "ind". - HeapWord* alloc_obj_from_region_index(int ind, size_t word_size); + // Finds a contiguous set of empty regions of length num, starting + // from a given index. + int find_contiguous_from(int from, size_t num); // Currently, we're choosing collection sets in a round-robin fashion, // starting here. @@ -76,11 +76,8 @@ // that are available for allocation. size_t free_suffix(); - // Requires "word_size" to be humongous (in the technical sense). If - // possible, allocates a contiguous subsequence of the heap regions to - // satisfy the allocation, and returns the address of the beginning of - // that sequence, otherwise returns NULL. - HeapWord* obj_allocate(size_t word_size); + // Finds a contiguous set of empty regions of length num. + int find_contiguous(size_t num); // Apply the "doHeapRegion" method of "blk" to all regions in "this", // in address order, terminating the iteration early @@ -106,7 +103,7 @@ // If "addr" falls within a region in the sequence, return that region, // or else NULL. - HeapRegion* addr_to_region(const void* addr); + inline HeapRegion* addr_to_region(const void* addr); void print(); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/heapRegionSet.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/hotspot/src/share/vm/gc_implementation/g1/heapRegionSet.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -0,0 +1,438 @@ +/* + * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc_implementation/g1/heapRegionSet.inline.hpp" + +size_t HeapRegionSetBase::_unrealistically_long_length = 0; + +//////////////////// HeapRegionSetBase //////////////////// + +void HeapRegionSetBase::set_unrealistically_long_length(size_t len) { + guarantee(_unrealistically_long_length == 0, "should only be set once"); + _unrealistically_long_length = len; +} + +size_t HeapRegionSetBase::calculate_region_num(HeapRegion* hr) { + assert(hr->startsHumongous(), "pre-condition"); + assert(hr->capacity() % HeapRegion::GrainBytes == 0, "invariant"); + size_t region_num = hr->capacity() >> HeapRegion::LogOfHRGrainBytes; + assert(region_num > 0, "sanity"); + return region_num; +} + +void HeapRegionSetBase::fill_in_ext_msg(hrl_ext_msg* msg, const char* message) { + msg->append("[%s] %s " + "ln: "SIZE_FORMAT" rn: "SIZE_FORMAT" " + "cy: "SIZE_FORMAT" ud: "SIZE_FORMAT, + name(), message, length(), region_num(), + total_capacity_bytes(), total_used_bytes()); + fill_in_ext_msg_extra(msg); +} + +bool HeapRegionSetBase::verify_region(HeapRegion* hr, + HeapRegionSetBase* expected_containing_set) { + const char* error_message = NULL; + + if (!regions_humongous()) { + if (hr->isHumongous()) { + error_message = "the region should not be humongous"; + } + } else { + if (!hr->isHumongous() || !hr->startsHumongous()) { + error_message = "the region should be 'starts humongous'"; + } + } + + if (!regions_empty()) { + if (hr->is_empty()) { + error_message = "the region should not be empty"; + } + } else { + if (!hr->is_empty()) { + error_message = "the region should be empty"; + } + } + +#ifdef ASSERT + // The _containing_set field is only available when ASSERT is defined. + if (hr->containing_set() != expected_containing_set) { + error_message = "inconsistent containing set found"; + } +#endif // ASSERT + + const char* extra_error_message = verify_region_extra(hr); + if (extra_error_message != NULL) { + error_message = extra_error_message; + } + + if (error_message != NULL) { + outputStream* out = tty; + out->cr(); + out->print_cr("## [%s] %s", name(), error_message); + out->print_cr("## Offending Region: "PTR_FORMAT, hr); + out->print_cr(" "HR_FORMAT, HR_FORMAT_PARAMS(hr)); +#ifdef ASSERT + out->print_cr(" containing set: "PTR_FORMAT, hr->containing_set()); +#endif // ASSERT + out->print_cr("## Offending Region Set: "PTR_FORMAT, this); + print_on(out); + return false; + } else { + return true; + } +} + +void HeapRegionSetBase::verify() { + // It's important that we also observe the MT safety protocol even + // for the verification calls. If we do verification without the + // appropriate locks and the set changes underneath our feet + // verification might fail and send us on a wild goose chase. + hrl_assert_mt_safety_ok(this); + + guarantee(( is_empty() && length() == 0 && region_num() == 0 && + total_used_bytes() == 0 && total_capacity_bytes() == 0) || + (!is_empty() && length() >= 0 && region_num() >= 0 && + total_used_bytes() >= 0 && total_capacity_bytes() >= 0), + hrl_ext_msg(this, "invariant")); + + guarantee((!regions_humongous() && region_num() == length()) || + ( regions_humongous() && region_num() >= length()), + hrl_ext_msg(this, "invariant")); + + guarantee(!regions_empty() || total_used_bytes() == 0, + hrl_ext_msg(this, "invariant")); + + guarantee(total_used_bytes() <= total_capacity_bytes(), + hrl_ext_msg(this, "invariant")); +} + +void HeapRegionSetBase::verify_start() { + // See comment in verify() about MT safety and verification. + hrl_assert_mt_safety_ok(this); + assert(!_verify_in_progress, + hrl_ext_msg(this, "verification should not be in progress")); + + // Do the basic verification first before we do the checks over the regions. + HeapRegionSetBase::verify(); + + _calc_length = 0; + _calc_region_num = 0; + _calc_total_capacity_bytes = 0; + _calc_total_used_bytes = 0; + _verify_in_progress = true; +} + +void HeapRegionSetBase::verify_next_region(HeapRegion* hr) { + // See comment in verify() about MT safety and verification. + hrl_assert_mt_safety_ok(this); + assert(_verify_in_progress, + hrl_ext_msg(this, "verification should be in progress")); + + guarantee(verify_region(hr, this), hrl_ext_msg(this, "region verification")); + + _calc_length += 1; + if (!hr->isHumongous()) { + _calc_region_num += 1; + } else { + _calc_region_num += calculate_region_num(hr); + } + _calc_total_capacity_bytes += hr->capacity(); + _calc_total_used_bytes += hr->used(); +} + +void HeapRegionSetBase::verify_end() { + // See comment in verify() about MT safety and verification. + hrl_assert_mt_safety_ok(this); + assert(_verify_in_progress, + hrl_ext_msg(this, "verification should be in progress")); + + guarantee(length() == _calc_length, + hrl_err_msg("[%s] length: "SIZE_FORMAT" should be == " + "calc length: "SIZE_FORMAT, + name(), length(), _calc_length)); + + guarantee(region_num() == _calc_region_num, + hrl_err_msg("[%s] region num: "SIZE_FORMAT" should be == " + "calc region num: "SIZE_FORMAT, + name(), region_num(), _calc_region_num)); + + guarantee(total_capacity_bytes() == _calc_total_capacity_bytes, + hrl_err_msg("[%s] capacity bytes: "SIZE_FORMAT" should be == " + "calc capacity bytes: "SIZE_FORMAT, + name(), + total_capacity_bytes(), _calc_total_capacity_bytes)); + + guarantee(total_used_bytes() == _calc_total_used_bytes, + hrl_err_msg("[%s] used bytes: "SIZE_FORMAT" should be == " + "calc used bytes: "SIZE_FORMAT, + name(), total_used_bytes(), _calc_total_used_bytes)); + + _verify_in_progress = false; +} + +void HeapRegionSetBase::print_on(outputStream* out, bool print_contents) { + out->cr(); + out->print_cr("Set: %s ("PTR_FORMAT")", name(), this); + out->print_cr(" Region Assumptions"); + out->print_cr(" humongous : %s", BOOL_TO_STR(regions_humongous())); + out->print_cr(" empty : %s", BOOL_TO_STR(regions_empty())); + out->print_cr(" Attributes"); + out->print_cr(" length : "SIZE_FORMAT_W(14), length()); + out->print_cr(" region num : "SIZE_FORMAT_W(14), region_num()); + out->print_cr(" total capacity : "SIZE_FORMAT_W(14)" bytes", + total_capacity_bytes()); + out->print_cr(" total used : "SIZE_FORMAT_W(14)" bytes", + total_used_bytes()); +} + +void HeapRegionSetBase::clear() { + _length = 0; + _region_num = 0; + _total_used_bytes = 0; +} + +HeapRegionSetBase::HeapRegionSetBase(const char* name) + : _name(name), _verify_in_progress(false), + _calc_length(0), _calc_region_num(0), + _calc_total_capacity_bytes(0), _calc_total_used_bytes(0) { } + +//////////////////// HeapRegionSet //////////////////// + +void HeapRegionSet::update_from_proxy(HeapRegionSet* proxy_set) { + hrl_assert_mt_safety_ok(this); + hrl_assert_mt_safety_ok(proxy_set); + hrl_assert_sets_match(this, proxy_set); + + verify_optional(); + proxy_set->verify_optional(); + + if (proxy_set->is_empty()) return; + + assert(proxy_set->length() <= _length, + hrl_err_msg("[%s] proxy set length: "SIZE_FORMAT" " + "should be <= length: "SIZE_FORMAT, + name(), proxy_set->length(), _length)); + _length -= proxy_set->length(); + + assert(proxy_set->region_num() <= _region_num, + hrl_err_msg("[%s] proxy set region num: "SIZE_FORMAT" " + "should be <= region num: "SIZE_FORMAT, + name(), proxy_set->region_num(), _region_num)); + _region_num -= proxy_set->region_num(); + + assert(proxy_set->total_used_bytes() <= _total_used_bytes, + hrl_err_msg("[%s] proxy set used bytes: "SIZE_FORMAT" " + "should be <= used bytes: "SIZE_FORMAT, + name(), proxy_set->total_used_bytes(), + _total_used_bytes)); + _total_used_bytes -= proxy_set->total_used_bytes(); + + proxy_set->clear(); + + verify_optional(); + proxy_set->verify_optional(); +} + +//////////////////// HeapRegionLinkedList //////////////////// + +void HeapRegionLinkedList::fill_in_ext_msg_extra(hrl_ext_msg* msg) { + msg->append(" hd: "PTR_FORMAT" tl: "PTR_FORMAT, head(), tail()); +} + +void HeapRegionLinkedList::add_as_tail(HeapRegionLinkedList* from_list) { + hrl_assert_mt_safety_ok(this); + hrl_assert_mt_safety_ok(from_list); + + verify_optional(); + from_list->verify_optional(); + + if (from_list->is_empty()) return; + +#ifdef ASSERT + HeapRegionLinkedListIterator iter(from_list); + while (iter.more_available()) { + HeapRegion* hr = iter.get_next(); + // In set_containing_set() we check that we either set the value + // from NULL to non-NULL or vice versa to catch bugs. So, we have + // to NULL it first before setting it to the value. + hr->set_containing_set(NULL); + hr->set_containing_set(this); + } +#endif // ASSERT + + if (_tail != NULL) { + assert(length() > 0 && _head != NULL, hrl_ext_msg(this, "invariant")); + _tail->set_next(from_list->_head); + } else { + assert(length() == 0 && _head == NULL, hrl_ext_msg(this, "invariant")); + _head = from_list->_head; + } + _tail = from_list->_tail; + + _length += from_list->length(); + _region_num += from_list->region_num(); + _total_used_bytes += from_list->total_used_bytes(); + from_list->clear(); + + verify_optional(); + from_list->verify_optional(); +} + +void HeapRegionLinkedList::remove_all() { + hrl_assert_mt_safety_ok(this); + verify_optional(); + + HeapRegion* curr = _head; + while (curr != NULL) { + hrl_assert_region_ok(this, curr, this); + + HeapRegion* next = curr->next(); + curr->set_next(NULL); + curr->set_containing_set(NULL); + curr = next; + } + clear(); + + verify_optional(); +} + +void HeapRegionLinkedList::remove_all_pending(size_t target_count) { + hrl_assert_mt_safety_ok(this); + assert(target_count > 1, hrl_ext_msg(this, "pre-condition")); + assert(!is_empty(), hrl_ext_msg(this, "pre-condition")); + + verify_optional(); + DEBUG_ONLY(size_t old_length = length();) + + HeapRegion* curr = _head; + HeapRegion* prev = NULL; + size_t count = 0; + while (curr != NULL) { + hrl_assert_region_ok(this, curr, this); + HeapRegion* next = curr->next(); + + if (curr->pending_removal()) { + assert(count < target_count, + hrl_err_msg("[%s] should not come across more regions " + "pending for removal than target_count: "SIZE_FORMAT, + name(), target_count)); + + if (prev == NULL) { + assert(_head == curr, hrl_ext_msg(this, "invariant")); + _head = next; + } else { + assert(_head != curr, hrl_ext_msg(this, "invariant")); + prev->set_next(next); + } + if (next == NULL) { + assert(_tail == curr, hrl_ext_msg(this, "invariant")); + _tail = prev; + } else { + assert(_tail != curr, hrl_ext_msg(this, "invariant")); + } + + curr->set_next(NULL); + remove_internal(curr); + curr->set_pending_removal(false); + + count += 1; + + // If we have come across the target number of regions we can + // just bail out. However, for debugging purposes, we can just + // carry on iterating to make sure there are not more regions + // tagged with pending removal. + DEBUG_ONLY(if (count == target_count) break;) + } else { + prev = curr; + } + curr = next; + } + + assert(count == target_count, + hrl_err_msg("[%s] count: "SIZE_FORMAT" should be == " + "target_count: "SIZE_FORMAT, name(), count, target_count)); + assert(length() + target_count == old_length, + hrl_err_msg("[%s] new length should be consistent " + "new length: "SIZE_FORMAT" old length: "SIZE_FORMAT" " + "target_count: "SIZE_FORMAT, + name(), length(), old_length, target_count)); + + verify_optional(); +} + +void HeapRegionLinkedList::verify() { + // See comment in HeapRegionSetBase::verify() about MT safety and + // verification. + hrl_assert_mt_safety_ok(this); + + // This will also do the basic verification too. + verify_start(); + + HeapRegion* curr = _head; + HeapRegion* prev1 = NULL; + HeapRegion* prev0 = NULL; + size_t count = 0; + while (curr != NULL) { + verify_next_region(curr); + + count += 1; + guarantee(count < _unrealistically_long_length, + hrl_err_msg("[%s] the calculated length: "SIZE_FORMAT" " + "seems very long, is there maybe a cycle? " + "curr: "PTR_FORMAT" prev0: "PTR_FORMAT" " + "prev1: "PTR_FORMAT" length: "SIZE_FORMAT, + name(), count, curr, prev0, prev1, length())); + + prev1 = prev0; + prev0 = curr; + curr = curr->next(); + } + + guarantee(_tail == prev0, hrl_ext_msg(this, "post-condition")); + + verify_end(); +} + +void HeapRegionLinkedList::clear() { + HeapRegionSetBase::clear(); + _head = NULL; + _tail = NULL; +} + +void HeapRegionLinkedList::print_on(outputStream* out, bool print_contents) { + HeapRegionSetBase::print_on(out, print_contents); + out->print_cr(" Linking"); + out->print_cr(" head : "PTR_FORMAT, _head); + out->print_cr(" tail : "PTR_FORMAT, _tail); + + if (print_contents) { + out->print_cr(" Contents"); + HeapRegionLinkedListIterator iter(this); + while (iter.more_available()) { + HeapRegion* hr = iter.get_next(); + hr->print_on(out); + } + } +} diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/heapRegionSet.hpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/hotspot/src/share/vm/gc_implementation/g1/heapRegionSet.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -0,0 +1,346 @@ +/* + * copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSET_HPP +#define SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSET_HPP + +#include "gc_implementation/g1/heapRegion.hpp" + +// Large buffer for some cases where the output might be larger than normal. +#define HRL_ERR_MSG_BUFSZ 512 +typedef FormatBuffer hrl_err_msg; + +// Set verification will be forced either if someone defines +// HEAP_REGION_SET_FORCE_VERIFY to be 1, or in builds in which +// asserts are compiled in. +#ifndef HEAP_REGION_SET_FORCE_VERIFY +#define HEAP_REGION_SET_FORCE_VERIFY defined(ASSERT) +#endif // HEAP_REGION_SET_FORCE_VERIFY + +//////////////////// HeapRegionSetBase //////////////////// + +// Base class for all the classes that represent heap region sets. It +// contains the basic attributes that each set needs to maintain +// (e.g., length, region num, used bytes sum) plus any shared +// functionality (e.g., verification). + +class hrl_ext_msg; + +class HeapRegionSetBase VALUE_OBJ_CLASS_SPEC { + friend class hrl_ext_msg; + +protected: + static size_t calculate_region_num(HeapRegion* hr); + + static size_t _unrealistically_long_length; + + // The number of regions added to the set. If the set contains + // only humongous regions, this reflects only 'starts humongous' + // regions and does not include 'continues humongous' ones. + size_t _length; + + // The total number of regions represented by the set. If the set + // does not contain humongous regions, this should be the same as + // _length. If the set contains only humongous regions, this will + // include the 'continues humongous' regions. + size_t _region_num; + + // We don't keep track of the total capacity explicitly, we instead + // recalculate it based on _region_num and the heap region size. + + // The sum of used bytes in the all the regions in the set. + size_t _total_used_bytes; + + const char* _name; + + bool _verify_in_progress; + size_t _calc_length; + size_t _calc_region_num; + size_t _calc_total_capacity_bytes; + size_t _calc_total_used_bytes; + + // verify_region() is used to ensure that the contents of a region + // added to / removed from a set are consistent. Different sets + // make different assumptions about the regions added to them. So + // each set can override verify_region_extra(), which is called + // from verify_region(), and do any extra verification it needs to + // perform in that. + virtual const char* verify_region_extra(HeapRegion* hr) { return NULL; } + bool verify_region(HeapRegion* hr, + HeapRegionSetBase* expected_containing_set); + + // Indicates whether all regions in the set should be humongous or + // not. Only used during verification. + virtual bool regions_humongous() = 0; + + // Indicates whether all regions in the set should be empty or + // not. Only used during verification. + virtual bool regions_empty() = 0; + + // Subclasses can optionally override this to do MT safety protocol + // checks. It is called in an assert from all methods that perform + // updates on the set (and subclasses should also call it too). + virtual bool check_mt_safety() { return true; } + + // fill_in_ext_msg() writes the the values of the set's attributes + // in the custom err_msg (hrl_ext_msg). fill_in_ext_msg_extra() + // allows subclasses to append further information. + virtual void fill_in_ext_msg_extra(hrl_ext_msg* msg) { } + void fill_in_ext_msg(hrl_ext_msg* msg, const char* message); + + // It updates the fields of the set to reflect hr being added to + // the set. + inline void update_for_addition(HeapRegion* hr); + + // It updates the fields of the set to reflect hr being added to + // the set and tags the region appropriately. + inline void add_internal(HeapRegion* hr); + + // It updates the fields of the set to reflect hr being removed + // from the set. + inline void update_for_removal(HeapRegion* hr); + + // It updates the fields of the set to reflect hr being removed + // from the set and tags the region appropriately. + inline void remove_internal(HeapRegion* hr); + + // It clears all the fields of the sets. Note: it will not iterate + // over the set and remove regions from it. It assumes that the + // caller has already done so. It will literally just clear the fields. + virtual void clear(); + + HeapRegionSetBase(const char* name); + +public: + static void set_unrealistically_long_length(size_t len); + + const char* name() { return _name; } + + size_t length() { return _length; } + + bool is_empty() { return _length == 0; } + + size_t region_num() { return _region_num; } + + size_t total_capacity_bytes() { + return region_num() << HeapRegion::LogOfHRGrainBytes; + } + + size_t total_used_bytes() { return _total_used_bytes; } + + virtual void verify(); + void verify_start(); + void verify_next_region(HeapRegion* hr); + void verify_end(); + +#if HEAP_REGION_SET_FORCE_VERIFY + void verify_optional() { + verify(); + } +#else // HEAP_REGION_SET_FORCE_VERIFY + void verify_optional() { } +#endif // HEAP_REGION_SET_FORCE_VERIFY + + virtual void print_on(outputStream* out, bool print_contents = false); +}; + +// Customized err_msg for heap region sets. Apart from a +// assert/guarantee-specific message it also prints out the values of +// the fields of the associated set. This can be very helpful in +// diagnosing failures. + +class hrl_ext_msg : public hrl_err_msg { +public: + hrl_ext_msg(HeapRegionSetBase* set, const char* message) : hrl_err_msg("") { + set->fill_in_ext_msg(this, message); + } +}; + +// These two macros are provided for convenience, to keep the uses of +// these two asserts a bit more concise. + +#define hrl_assert_mt_safety_ok(_set_) \ + do { \ + assert((_set_)->check_mt_safety(), hrl_ext_msg((_set_), "MT safety")); \ + } while (0) + +#define hrl_assert_region_ok(_set_, _hr_, _expected_) \ + do { \ + assert((_set_)->verify_region((_hr_), (_expected_)), \ + hrl_ext_msg((_set_), "region verification")); \ + } while (0) + +//////////////////// HeapRegionSet //////////////////// + +#define hrl_assert_sets_match(_set1_, _set2_) \ + do { \ + assert(((_set1_)->regions_humongous() == \ + (_set2_)->regions_humongous()) && \ + ((_set1_)->regions_empty() == (_set2_)->regions_empty()), \ + hrl_err_msg("the contents of set %s and set %s should match", \ + (_set1_)->name(), (_set2_)->name())); \ + } while (0) + +// This class represents heap region sets whose members are not +// explicitly tracked. It's helpful to group regions using such sets +// so that we can reason about all the region groups in the heap using +// the same interface (namely, the HeapRegionSetBase API). + +class HeapRegionSet : public HeapRegionSetBase { +protected: + virtual const char* verify_region_extra(HeapRegion* hr) { + if (hr->next() != NULL) { + return "next() should always be NULL as we do not link the regions"; + } + + return HeapRegionSetBase::verify_region_extra(hr); + } + + HeapRegionSet(const char* name) : HeapRegionSetBase(name) { + clear(); + } + +public: + // It adds hr to the set. The region should not be a member of + // another set. + inline void add(HeapRegion* hr); + + // It removes hr from the set. The region should be a member of + // this set. + inline void remove(HeapRegion* hr); + + // It removes a region from the set. Instead of updating the fields + // of the set to reflect this removal, it accumulates the updates + // in proxy_set. The idea is that proxy_set is thread-local to + // avoid multiple threads updating the fields of the set + // concurrently and having to synchronize. The method + // update_from_proxy() will update the fields of the set from the + // proxy_set. + inline void remove_with_proxy(HeapRegion* hr, HeapRegionSet* proxy_set); + + // After multiple calls to remove_with_proxy() the updates to the + // fields of the set are accumulated in proxy_set. This call + // updates the fields of the set from proxy_set. + void update_from_proxy(HeapRegionSet* proxy_set); +}; + +//////////////////// HeapRegionLinkedList //////////////////// + +// A set that links all the regions added to it in a singly-linked +// list. We should try to avoid doing operations that iterate over +// such lists in performance critical paths. Typically we should +// add / remove one region at a time or concatenate two lists. All +// those operations are done in constant time. + +class HeapRegionLinkedListIterator; + +class HeapRegionLinkedList : public HeapRegionSetBase { + friend class HeapRegionLinkedListIterator; + +private: + HeapRegion* _head; + HeapRegion* _tail; + + // These are provided for use by the friend classes. + HeapRegion* head() { return _head; } + HeapRegion* tail() { return _tail; } + +protected: + virtual void fill_in_ext_msg_extra(hrl_ext_msg* msg); + + // See the comment for HeapRegionSetBase::clear() + virtual void clear(); + + HeapRegionLinkedList(const char* name) : HeapRegionSetBase(name) { + clear(); + } + +public: + // It adds hr to the list as the new tail. The region should not be + // a member of another set. + inline void add_as_tail(HeapRegion* hr); + + // It removes and returns the head of the list. It assumes that the + // list is not empty so it will return a non-NULL value. + inline HeapRegion* remove_head(); + + // Convenience method. + inline HeapRegion* remove_head_or_null(); + + // It moves the regions from from_list to this list and empties + // from_list. The new regions will appear in the same order as they + // were in from_list and be linked in the end of this list. + void add_as_tail(HeapRegionLinkedList* from_list); + + // It empties the list by removing all regions from it. + void remove_all(); + + // It removes all regions in the list that are pending for removal + // (i.e., they have been tagged with "pending_removal"). The list + // must not be empty, target_count should reflect the exact number + // of regions that are pending for removal in the list, and + // target_count should be > 1 (currently, we never need to remove a + // single region using this). + void remove_all_pending(size_t target_count); + + virtual void verify(); + + virtual void print_on(outputStream* out, bool print_contents = false); +}; + +//////////////////// HeapRegionLinkedList //////////////////// + +// Iterator class that provides a convenient way to iterator over the +// regions in a HeapRegionLinkedList instance. + +class HeapRegionLinkedListIterator : public StackObj { +private: + HeapRegionLinkedList* _list; + HeapRegion* _curr; + +public: + bool more_available() { + return _curr != NULL; + } + + HeapRegion* get_next() { + assert(more_available(), + "get_next() should be called when more regions are available"); + + // If we are going to introduce a count in the iterator we should + // do the "cycle" check. + + HeapRegion* hr = _curr; + assert(_list->verify_region(hr, _list), "region verification"); + _curr = hr->next(); + return hr; + } + + HeapRegionLinkedListIterator(HeapRegionLinkedList* list) + : _curr(NULL), _list(list) { + _curr = list->head(); + } +}; + +#endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSET_HPP diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/heapRegionSet.inline.hpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/hotspot/src/share/vm/gc_implementation/g1/heapRegionSet.inline.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -0,0 +1,159 @@ +/* + * copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSET_INLINE_HPP +#define SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSET_INLINE_HPP + +#include "gc_implementation/g1/heapRegionSet.hpp" + +//////////////////// HeapRegionSetBase //////////////////// + +inline void HeapRegionSetBase::update_for_addition(HeapRegion* hr) { + // Assumes the caller has already verified the region. + + _length += 1; + if (!hr->isHumongous()) { + _region_num += 1; + } else { + _region_num += calculate_region_num(hr); + } + _total_used_bytes += hr->used(); +} + +inline void HeapRegionSetBase::add_internal(HeapRegion* hr) { + hrl_assert_region_ok(this, hr, NULL); + assert(hr->next() == NULL, hrl_ext_msg(this, "should not already be linked")); + + update_for_addition(hr); + hr->set_containing_set(this); +} + +inline void HeapRegionSetBase::update_for_removal(HeapRegion* hr) { + // Assumes the caller has already verified the region. + assert(_length > 0, hrl_ext_msg(this, "pre-condition")); + _length -= 1; + + size_t region_num_diff; + if (!hr->isHumongous()) { + region_num_diff = 1; + } else { + region_num_diff = calculate_region_num(hr); + } + assert(region_num_diff <= _region_num, + hrl_err_msg("[%s] region's region num: "SIZE_FORMAT" " + "should be <= region num: "SIZE_FORMAT, + name(), region_num_diff, _region_num)); + _region_num -= region_num_diff; + + size_t used_bytes = hr->used(); + assert(used_bytes <= _total_used_bytes, + hrl_err_msg("[%s] region's used bytes: "SIZE_FORMAT" " + "should be <= used bytes: "SIZE_FORMAT, + name(), used_bytes, _total_used_bytes)); + _total_used_bytes -= used_bytes; +} + +inline void HeapRegionSetBase::remove_internal(HeapRegion* hr) { + hrl_assert_region_ok(this, hr, this); + assert(hr->next() == NULL, hrl_ext_msg(this, "should already be unlinked")); + + hr->set_containing_set(NULL); + update_for_removal(hr); +} + +//////////////////// HeapRegionSet //////////////////// + +inline void HeapRegionSet::add(HeapRegion* hr) { + hrl_assert_mt_safety_ok(this); + // add_internal() will verify the region. + add_internal(hr); +} + +inline void HeapRegionSet::remove(HeapRegion* hr) { + hrl_assert_mt_safety_ok(this); + // remove_internal() will verify the region. + remove_internal(hr); +} + +inline void HeapRegionSet::remove_with_proxy(HeapRegion* hr, + HeapRegionSet* proxy_set) { + // No need to fo the MT safety check here given that this method + // does not update the contents of the set but instead accumulates + // the changes in proxy_set which is assumed to be thread-local. + hrl_assert_sets_match(this, proxy_set); + hrl_assert_region_ok(this, hr, this); + + hr->set_containing_set(NULL); + proxy_set->update_for_addition(hr); +} + +//////////////////// HeapRegionLinkedList //////////////////// + +inline void HeapRegionLinkedList::add_as_tail(HeapRegion* hr) { + hrl_assert_mt_safety_ok(this); + assert((length() == 0 && _head == NULL && _tail == NULL) || + (length() > 0 && _head != NULL && _tail != NULL), + hrl_ext_msg(this, "invariant")); + // add_internal() will verify the region. + add_internal(hr); + + // Now link the region. + if (_tail != NULL) { + _tail->set_next(hr); + } else { + _head = hr; + } + _tail = hr; +} + +inline HeapRegion* HeapRegionLinkedList::remove_head() { + hrl_assert_mt_safety_ok(this); + assert(!is_empty(), hrl_ext_msg(this, "the list should not be empty")); + assert(length() > 0 && _head != NULL && _tail != NULL, + hrl_ext_msg(this, "invariant")); + + // We need to unlink it first. + HeapRegion* hr = _head; + _head = hr->next(); + if (_head == NULL) { + _tail = NULL; + } + hr->set_next(NULL); + + // remove_internal() will verify the region. + remove_internal(hr); + return hr; +} + +inline HeapRegion* HeapRegionLinkedList::remove_head_or_null() { + hrl_assert_mt_safety_ok(this); + + if (!is_empty()) { + return remove_head(); + } else { + return NULL; + } +} + +#endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSET_INLINE_HPP diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/heapRegionSets.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/hotspot/src/share/vm/gc_implementation/g1/heapRegionSets.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc_implementation/g1/heapRegionSets.hpp" + +//////////////////// FreeRegionList //////////////////// + +const char* FreeRegionList::verify_region_extra(HeapRegion* hr) { + if (hr->is_young()) { + return "the region should not be young"; + } + // The superclass will check that the region is empty and + // not-humongous. + return HeapRegionLinkedList::verify_region_extra(hr); +} + +//////////////////// MasterFreeRegionList //////////////////// + +bool MasterFreeRegionList::check_mt_safety() { + // Master Free List MT safety protocol: + // (a) If we're at a safepoint, operations on the master free list + // should be invoked by either the VM thread (which will serialize + // them) or by the GC workers while holding the + // FreeList_lock. + // (b) If we're not at a safepoint, operations on the master free + // list should be invoked while holding the Heap_lock. + + guarantee((SafepointSynchronize::is_at_safepoint() && + (Thread::current()->is_VM_thread() || + FreeList_lock->owned_by_self())) || + (!SafepointSynchronize::is_at_safepoint() && + Heap_lock->owned_by_self()), + hrl_ext_msg(this, "master free list MT safety protocol")); + + return FreeRegionList::check_mt_safety(); +} + +//////////////////// SecondaryFreeRegionList //////////////////// + +bool SecondaryFreeRegionList::check_mt_safety() { + // Secondary Free List MT safety protocol: + // Operations on the secondary free list should always be invoked + // while holding the SecondaryFreeList_lock. + + guarantee(SecondaryFreeList_lock->owned_by_self(), + hrl_ext_msg(this, "secondary free list MT safety protocol")); + + return FreeRegionList::check_mt_safety(); +} + +//////////////////// HumongousRegionSet //////////////////// + +const char* HumongousRegionSet::verify_region_extra(HeapRegion* hr) { + if (hr->is_young()) { + return "the region should not be young"; + } + // The superclass will check that the region is not empty and + // humongous. + return HeapRegionSet::verify_region_extra(hr); +} + +//////////////////// HumongousRegionSet //////////////////// + +bool MasterHumongousRegionSet::check_mt_safety() { + // Master Humongous Set MT safety protocol: + // (a) If we're at a safepoint, operations on the master humongous + // set should be invoked by either the VM thread (which will + // serialize them) or by the GC workers while holding the + // OldSets_lock. + // (b) If we're not at a safepoint, operations on the master + // humongous set should be invoked while holding the Heap_lock. + + guarantee((SafepointSynchronize::is_at_safepoint() && + (Thread::current()->is_VM_thread() || + OldSets_lock->owned_by_self())) || + (!SafepointSynchronize::is_at_safepoint() && + Heap_lock->owned_by_self()), + hrl_ext_msg(this, "master humongous set MT safety protocol")); + return HumongousRegionSet::check_mt_safety(); +} diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/heapRegionSets.hpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/hotspot/src/share/vm/gc_implementation/g1/heapRegionSets.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -0,0 +1,86 @@ +/* + * copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSETS_HPP +#define SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSETS_HPP + +#include "gc_implementation/g1/heapRegionSet.inline.hpp" + +//////////////////// FreeRegionList //////////////////// + +class FreeRegionList : public HeapRegionLinkedList { +protected: + virtual const char* verify_region_extra(HeapRegion* hr); + + virtual bool regions_humongous() { return false; } + virtual bool regions_empty() { return true; } + +public: + FreeRegionList(const char* name) : HeapRegionLinkedList(name) { } +}; + +//////////////////// MasterFreeRegionList //////////////////// + +class MasterFreeRegionList : public FreeRegionList { +protected: + virtual bool check_mt_safety(); + +public: + MasterFreeRegionList(const char* name) : FreeRegionList(name) { } +}; + +//////////////////// SecondaryFreeRegionList //////////////////// + +class SecondaryFreeRegionList : public FreeRegionList { +protected: + virtual bool check_mt_safety(); + +public: + SecondaryFreeRegionList(const char* name) : FreeRegionList(name) { } +}; + +//////////////////// HumongousRegionSet //////////////////// + +class HumongousRegionSet : public HeapRegionSet { +protected: + virtual const char* verify_region_extra(HeapRegion* hr); + + virtual bool regions_humongous() { return true; } + virtual bool regions_empty() { return false; } + +public: + HumongousRegionSet(const char* name) : HeapRegionSet(name) { } +}; + +//////////////////// MasterHumongousRegionSet //////////////////// + +class MasterHumongousRegionSet : public HumongousRegionSet { +protected: + virtual bool check_mt_safety(); + +public: + MasterHumongousRegionSet(const char* name) : HumongousRegionSet(name) { } +}; + +#endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSETS_HPP diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/ptrQueue.cpp --- a/hotspot/src/share/vm/gc_implementation/g1/ptrQueue.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/ptrQueue.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -38,8 +38,8 @@ # include "thread_windows.inline.hpp" #endif -PtrQueue::PtrQueue(PtrQueueSet* qset_, bool perm, bool active) : - _qset(qset_), _buf(NULL), _index(0), _active(active), +PtrQueue::PtrQueue(PtrQueueSet* qset, bool perm, bool active) : + _qset(qset), _buf(NULL), _index(0), _active(active), _perm(perm), _lock(NULL) {} @@ -153,10 +153,16 @@ } void PtrQueue::handle_zero_index() { - assert(0 == _index, "Precondition."); + assert(_index == 0, "Precondition."); + // This thread records the full buffer and allocates a new one (while // holding the lock if there is one). if (_buf != NULL) { + if (!should_enqueue_buffer()) { + assert(_index > 0, "the buffer can only be re-used if it's not full"); + return; + } + if (_lock) { assert(_lock->owned_by_self(), "Required."); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/ptrQueue.hpp --- a/hotspot/src/share/vm/gc_implementation/g1/ptrQueue.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/ptrQueue.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -68,7 +68,7 @@ public: // Initialize this queue to contain a null buffer, and be part of the // given PtrQueueSet. - PtrQueue(PtrQueueSet*, bool perm = false, bool active = false); + PtrQueue(PtrQueueSet* qset, bool perm = false, bool active = false); // Release any contained resources. void flush(); // Calls flush() when destroyed. @@ -85,6 +85,14 @@ else enqueue_known_active(ptr); } + // This method is called when we're doing the zero index handling + // and gives a chance to the queues to do any pre-enqueueing + // processing they might want to do on the buffer. It should return + // true if the buffer should be enqueued, or false if enough + // entries were cleared from it so that it can be re-used. It should + // not return false if the buffer is still full (otherwise we can + // get into an infinite loop). + virtual bool should_enqueue_buffer() { return true; } void handle_zero_index(); void locking_enqueue_completed_buffer(void** buf); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/satbQueue.cpp --- a/hotspot/src/share/vm/gc_implementation/g1/satbQueue.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/satbQueue.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,12 +23,98 @@ */ #include "precompiled.hpp" +#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" #include "gc_implementation/g1/satbQueue.hpp" #include "memory/allocation.inline.hpp" #include "memory/sharedHeap.hpp" #include "runtime/mutexLocker.hpp" #include "runtime/thread.hpp" +// This method removes entries from an SATB buffer that will not be +// useful to the concurrent marking threads. An entry is removed if it +// satisfies one of the following conditions: +// +// * it points to an object outside the G1 heap (G1's concurrent +// marking only visits objects inside the G1 heap), +// * it points to an object that has been allocated since marking +// started (according to SATB those objects do not need to be +// visited during marking), or +// * it points to an object that has already been marked (no need to +// process it again). +// +// The rest of the entries will be retained and are compacted towards +// the top of the buffer. If with this filtering we clear a large +// enough chunk of the buffer we can re-use it (instead of enqueueing +// it) and we can just allow the mutator to carry on executing. + +bool ObjPtrQueue::should_enqueue_buffer() { + assert(_lock == NULL || _lock->owned_by_self(), + "we should have taken the lock before calling this"); + + // A value of 0 means "don't filter SATB buffers". + if (G1SATBBufferEnqueueingThresholdPercent == 0) { + return true; + } + + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + // This method should only be called if there is a non-NULL buffer + // that is full. + assert(_index == 0, "pre-condition"); + assert(_buf != NULL, "pre-condition"); + + void** buf = _buf; + size_t sz = _sz; + + // Used for sanity checking at the end of the loop. + debug_only(size_t entries = 0; size_t retained = 0;) + + size_t i = sz; + size_t new_index = sz; + + // Given that we are expecting _index == 0, we could have changed + // the loop condition to (i > 0). But we are using _index for + // generality. + while (i > _index) { + assert(i > 0, "we should have at least one more entry to process"); + i -= oopSize; + debug_only(entries += 1;) + oop* p = (oop*) &buf[byte_index_to_index((int) i)]; + oop obj = *p; + // NULL the entry so that unused parts of the buffer contain NULLs + // at the end. If we are going to retain it we will copy it to its + // final place. If we have retained all entries we have visited so + // far, we'll just end up copying it to the same place. + *p = NULL; + + bool retain = g1h->is_obj_ill(obj); + if (retain) { + assert(new_index > 0, "we should not have already filled up the buffer"); + new_index -= oopSize; + assert(new_index >= i, + "new_index should never be below i, as we alwaysr compact 'up'"); + oop* new_p = (oop*) &buf[byte_index_to_index((int) new_index)]; + assert(new_p >= p, "the destination location should never be below " + "the source as we always compact 'up'"); + assert(*new_p == NULL, + "we should have already cleared the destination location"); + *new_p = obj; + debug_only(retained += 1;) + } + } + size_t entries_calc = (sz - _index) / oopSize; + assert(entries == entries_calc, "the number of entries we counted " + "should match the number of entries we calculated"); + size_t retained_calc = (sz - new_index) / oopSize; + assert(retained == retained_calc, "the number of retained entries we counted " + "should match the number of retained entries we calculated"); + size_t perc = retained_calc * 100 / entries_calc; + bool should_enqueue = perc > (size_t) G1SATBBufferEnqueueingThresholdPercent; + _index = new_index; + + return should_enqueue; +} + void ObjPtrQueue::apply_closure(ObjectClosure* cl) { if (_buf != NULL) { apply_closure_to_buffer(cl, _buf, _index, _sz); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/satbQueue.hpp --- a/hotspot/src/share/vm/gc_implementation/g1/satbQueue.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/satbQueue.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -33,13 +33,18 @@ // A ptrQueue whose elements are "oops", pointers to object heads. class ObjPtrQueue: public PtrQueue { public: - ObjPtrQueue(PtrQueueSet* qset_, bool perm = false) : + ObjPtrQueue(PtrQueueSet* qset, bool perm = false) : // SATB queues are only active during marking cycles. We create // them with their active field set to false. If a thread is // created during a cycle and its SATB queue needs to be activated // before the thread starts running, we'll need to set its active // field to true. This is done in JavaThread::initialize_queues(). - PtrQueue(qset_, perm, false /* active */) { } + PtrQueue(qset, perm, false /* active */) { } + + // Overrides PtrQueue::should_enqueue_buffer(). See the method's + // definition for more information. + virtual bool should_enqueue_buffer(); + // Apply the closure to all elements, and reset the index to make the // buffer empty. void apply_closure(ObjectClosure* cl); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/g1/vm_operations_g1.cpp --- a/hotspot/src/share/vm/gc_implementation/g1/vm_operations_g1.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/g1/vm_operations_g1.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -38,7 +38,6 @@ } void VM_G1CollectForAllocation::doit() { - JvmtiGCForAllocationMarker jgcm; G1CollectedHeap* g1h = G1CollectedHeap::heap(); _result = g1h->satisfy_failed_allocation(_word_size, &_pause_succeeded); assert(_result == NULL || _pause_succeeded, @@ -46,7 +45,6 @@ } void VM_G1CollectFull::doit() { - JvmtiGCFullMarker jgcm; G1CollectedHeap* g1h = G1CollectedHeap::heap(); GCCauseSetter x(g1h, _gc_cause); g1h->do_full_collection(false /* clear_all_soft_refs */); @@ -72,7 +70,6 @@ } void VM_G1IncCollectionPause::doit() { - JvmtiGCForAllocationMarker jgcm; G1CollectedHeap* g1h = G1CollectedHeap::heap(); assert(!_should_initiate_conc_mark || ((_gc_cause == GCCause::_gc_locker && GCLockerInvokesConcurrent) || diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/parallelScavenge/psMarkSweep.cpp --- a/hotspot/src/share/vm/gc_implementation/parallelScavenge/psMarkSweep.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/parallelScavenge/psMarkSweep.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -258,6 +258,7 @@ BiasedLocking::restore_marks(); Threads::gc_epilogue(); CodeCache::gc_epilogue(); + JvmtiExport::gc_epilogue(); COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/parallelScavenge/psParallelCompact.cpp --- a/hotspot/src/share/vm/gc_implementation/parallelScavenge/psParallelCompact.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/parallelScavenge/psParallelCompact.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1054,6 +1054,7 @@ Threads::gc_epilogue(); CodeCache::gc_epilogue(); + JvmtiExport::gc_epilogue(); COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/parallelScavenge/vmPSOperations.cpp --- a/hotspot/src/share/vm/gc_implementation/parallelScavenge/vmPSOperations.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/parallelScavenge/vmPSOperations.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -42,8 +42,7 @@ } void VM_ParallelGCFailedAllocation::doit() { - JvmtiGCForAllocationMarker jgcm; - notify_gc_begin(false); + SvcGCMarker sgcm(SvcGCMarker::MINOR); ParallelScavengeHeap* heap = (ParallelScavengeHeap*)Universe::heap(); assert(heap->kind() == CollectedHeap::ParallelScavengeHeap, "must be a ParallelScavengeHeap"); @@ -54,8 +53,6 @@ if (_result == NULL && GC_locker::is_active_and_needs_gc()) { set_gc_locked(); } - - notify_gc_end(); } VM_ParallelGCFailedPermanentAllocation::VM_ParallelGCFailedPermanentAllocation(size_t size, @@ -67,8 +64,7 @@ } void VM_ParallelGCFailedPermanentAllocation::doit() { - JvmtiGCFullMarker jgcm; - notify_gc_begin(true); + SvcGCMarker sgcm(SvcGCMarker::FULL); ParallelScavengeHeap* heap = (ParallelScavengeHeap*)Universe::heap(); assert(heap->kind() == CollectedHeap::ParallelScavengeHeap, "must be a ParallelScavengeHeap"); @@ -78,7 +74,6 @@ if (_result == NULL && GC_locker::is_active_and_needs_gc()) { set_gc_locked(); } - notify_gc_end(); } // Only used for System.gc() calls @@ -91,8 +86,7 @@ } void VM_ParallelGCSystemGC::doit() { - JvmtiGCFullMarker jgcm; - notify_gc_begin(true); + SvcGCMarker sgcm(SvcGCMarker::FULL); ParallelScavengeHeap* heap = (ParallelScavengeHeap*)Universe::heap(); assert(heap->kind() == CollectedHeap::ParallelScavengeHeap, @@ -106,5 +100,4 @@ } else { heap->invoke_full_gc(false); } - notify_gc_end(); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/shared/vmGCOperations.cpp --- a/hotspot/src/share/vm/gc_implementation/shared/vmGCOperations.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/shared/vmGCOperations.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -31,7 +31,6 @@ #include "memory/oopFactory.hpp" #include "oops/instanceKlass.hpp" #include "oops/instanceRefKlass.hpp" -#include "prims/jvmtiExport.hpp" #include "runtime/handles.inline.hpp" #include "runtime/init.hpp" #include "runtime/interfaceSupport.hpp" @@ -40,6 +39,7 @@ #ifndef SERIALGC #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" #endif + HS_DTRACE_PROBE_DECL1(hotspot, gc__begin, bool); HS_DTRACE_PROBE_DECL(hotspot, gc__end); @@ -158,8 +158,7 @@ void VM_GenCollectForAllocation::doit() { - JvmtiGCForAllocationMarker jgcm; - notify_gc_begin(false); + SvcGCMarker sgcm(SvcGCMarker::MINOR); GenCollectedHeap* gch = GenCollectedHeap::heap(); GCCauseSetter gccs(gch, _gc_cause); @@ -169,22 +168,19 @@ if (_res == NULL && GC_locker::is_active_and_needs_gc()) { set_gc_locked(); } - notify_gc_end(); } void VM_GenCollectFull::doit() { - JvmtiGCFullMarker jgcm; - notify_gc_begin(true); + SvcGCMarker sgcm(SvcGCMarker::FULL); GenCollectedHeap* gch = GenCollectedHeap::heap(); GCCauseSetter gccs(gch, _gc_cause); gch->do_full_collection(gch->must_clear_all_soft_refs(), _max_level); - notify_gc_end(); } void VM_GenCollectForPermanentAllocation::doit() { - JvmtiGCForAllocationMarker jgcm; - notify_gc_begin(true); + SvcGCMarker sgcm(SvcGCMarker::FULL); + SharedHeap* heap = (SharedHeap*)Universe::heap(); GCCauseSetter gccs(heap, _gc_cause); switch (heap->kind()) { @@ -209,5 +205,4 @@ if (_res == NULL && GC_locker::is_active_and_needs_gc()) { set_gc_locked(); } - notify_gc_end(); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/gc_implementation/shared/vmGCOperations.hpp --- a/hotspot/src/share/vm/gc_implementation/shared/vmGCOperations.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/gc_implementation/shared/vmGCOperations.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -30,6 +30,7 @@ #include "runtime/jniHandles.hpp" #include "runtime/synchronizer.hpp" #include "runtime/vm_operations.hpp" +#include "prims/jvmtiExport.hpp" // The following class hierarchy represents // a set of operations (VM_Operation) related to GC. @@ -209,13 +210,17 @@ HeapWord* result() const { return _res; } }; -class DTraceGCProbeMarker : public StackObj { -public: - DTraceGCProbeMarker(bool full) { - VM_GC_Operation::notify_gc_begin(full); +class SvcGCMarker : public StackObj { + private: + JvmtiGCMarker _jgcm; + public: + typedef enum { MINOR, FULL, OTHER } reason_type; + + SvcGCMarker(reason_type reason ) { + VM_GC_Operation::notify_gc_begin(reason == FULL); } - ~DTraceGCProbeMarker() { + ~SvcGCMarker() { VM_GC_Operation::notify_gc_end(); } }; diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/interpreter/bytecode.cpp --- a/hotspot/src/share/vm/interpreter/bytecode.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/interpreter/bytecode.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -34,30 +34,6 @@ // Implementation of Bytecode -bool Bytecode::check_must_rewrite(Bytecodes::Code code) const { - assert(Bytecodes::can_rewrite(code), "post-check only"); - - // Some codes are conditionally rewriting. Look closely at them. - switch (code) { - case Bytecodes::_aload_0: - // Even if RewriteFrequentPairs is turned on, - // the _aload_0 code might delay its rewrite until - // a following _getfield rewrites itself. - return false; - - case Bytecodes::_lookupswitch: - return false; // the rewrite is not done by the interpreter - - case Bytecodes::_new: - // (Could actually look at the class here, but the profit would be small.) - return false; // the rewrite is not always done - } - - // No other special cases. - return true; -} - - #ifdef ASSERT void Bytecode::assert_same_format_as(Bytecodes::Code testbc, bool is_wide) const { @@ -188,17 +164,16 @@ // Note: Rewriter::rewrite changes the Java_u2 of an invokedynamic to a native_u4, // at the same time it allocates per-call-site CP cache entries. Bytecodes::Code rawc = code(); - Bytecode* invoke = bytecode(); - if (invoke->has_index_u4(rawc)) - return invoke->get_index_u4(rawc); + if (has_index_u4(rawc)) + return get_index_u4(rawc); else - return invoke->get_index_u2_cpcache(rawc); + return get_index_u2_cpcache(rawc); } int Bytecode_member_ref::pool_index() const { int index = this->index(); DEBUG_ONLY({ - if (!bytecode()->has_index_u4(code())) + if (!has_index_u4(code())) index -= constantPoolOopDesc::CPCACHE_INDEX_TAG; }); return _method->constants()->cache()->entry_at(index)->constant_pool_index(); @@ -214,13 +189,12 @@ // Implementation of Bytecode_loadconstant int Bytecode_loadconstant::raw_index() const { - Bytecode* bcp = bytecode(); - Bytecodes::Code rawc = bcp->code(); + Bytecodes::Code rawc = code(); assert(rawc != Bytecodes::_wide, "verifier prevents this"); if (Bytecodes::java_code(rawc) == Bytecodes::_ldc) - return bcp->get_index_u1(rawc); + return get_index_u1(rawc); else - return bcp->get_index_u2(rawc, false); + return get_index_u2(rawc, false); } int Bytecode_loadconstant::pool_index() const { @@ -258,7 +232,7 @@ case Bytecodes::_lookupswitch: { int i = number_of_pairs() - 1; while (i-- > 0) { - assert(pair_at(i)->match() < pair_at(i+1)->match(), "unsorted table entries"); + assert(pair_at(i).match() < pair_at(i+1).match(), "unsorted table entries"); } } break; diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/interpreter/bytecode.hpp --- a/hotspot/src/share/vm/interpreter/bytecode.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/interpreter/bytecode.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -38,14 +38,20 @@ # include "bytes_zero.hpp" #endif -// Base class for different kinds of abstractions working -// relative to an objects 'this' pointer. +class ciBytecodeStream; + +// The base class for different kinds of bytecode abstractions. +// Provides the primitive operations to manipulate code relative +// to the bcp. -class ThisRelativeObj VALUE_OBJ_CLASS_SPEC { - public: +class Bytecode: public StackObj { + protected: + const address _bcp; + const Bytecodes::Code _code; + // Address computation - address addr_at (int offset) const { return (address)this + offset; } - int byte_at (int offset) const { return *(addr_at(offset)); } + address addr_at (int offset) const { return (address)_bcp + offset; } + u_char byte_at(int offset) const { return *addr_at(offset); } address aligned_addr_at (int offset) const { return (address)round_to((intptr_t)addr_at(offset), jintSize); } int aligned_offset (int offset) const { return aligned_addr_at(offset) - addr_at(0); } @@ -54,31 +60,20 @@ int get_Java_u4_at (int offset) const { return Bytes::get_Java_u4(addr_at(offset)); } int get_native_u2_at (int offset) const { return Bytes::get_native_u2(addr_at(offset)); } int get_native_u4_at (int offset) const { return Bytes::get_native_u4(addr_at(offset)); } -}; - - -// The base class for different kinds of bytecode abstractions. -// Provides the primitive operations to manipulate code relative -// to an objects 'this' pointer. -// FIXME: Make this a ResourceObj, include the enclosing methodOop, and cache the opcode. - -class Bytecode: public ThisRelativeObj { - protected: - u_char byte_at(int offset) const { return *addr_at(offset); } - bool check_must_rewrite(Bytecodes::Code bc) const; public: - // Attributes - address bcp() const { return addr_at(0); } - int instruction_size() const { return Bytecodes::length_at(bcp()); } + Bytecode(methodOop method, address bcp): _bcp(bcp), _code(Bytecodes::code_at(method, addr_at(0))) { + assert(method != NULL, "this form requires a valid methodOop"); + } + // Defined in ciStreams.hpp + inline Bytecode(const ciBytecodeStream* stream, address bcp = NULL); - // Warning: Use code() with caution on live bytecode streams. 4926272 - Bytecodes::Code code() const { return Bytecodes::code_at(addr_at(0)); } + // Attributes + address bcp() const { return _bcp; } + int instruction_size() const { return Bytecodes::length_for_code_at(_code, bcp()); } + + Bytecodes::Code code() const { return _code; } Bytecodes::Code java_code() const { return Bytecodes::java_code(code()); } - bool must_rewrite(Bytecodes::Code code) const { return Bytecodes::can_rewrite(code) && check_must_rewrite(code); } - - // Creation - inline friend Bytecode* Bytecode_at(address bcp); // Static functions for parsing bytecodes in place. int get_index_u1(Bytecodes::Code bc) const { @@ -89,7 +84,7 @@ assert_same_format_as(bc, is_wide); assert_index_size(2, bc, is_wide); address p = addr_at(is_wide ? 2 : 1); if (can_use_native_byte_order(bc, is_wide)) - return Bytes::get_native_u2(p); + return Bytes::get_native_u2(p); else return Bytes::get_Java_u2(p); } int get_index_u1_cpcache(Bytecodes::Code bc) const { @@ -138,20 +133,17 @@ } }; -inline Bytecode* Bytecode_at(address bcp) { - // Warning: Use with caution on live bytecode streams. 4926272 - return (Bytecode*)bcp; -} - // Abstractions for lookupswitch bytecode - -class LookupswitchPair: ThisRelativeObj { +class LookupswitchPair VALUE_OBJ_CLASS_SPEC { private: - int _match; - int _offset; + const address _bcp; + + address addr_at (int offset) const { return _bcp + offset; } + int get_Java_u4_at (int offset) const { return Bytes::get_Java_u4(addr_at(offset)); } public: + LookupswitchPair(address bcp): _bcp(bcp) {} int match() const { return get_Java_u4_at(0 * jintSize); } int offset() const { return get_Java_u4_at(1 * jintSize); } }; @@ -159,26 +151,25 @@ class Bytecode_lookupswitch: public Bytecode { public: + Bytecode_lookupswitch(methodOop method, address bcp): Bytecode(method, bcp) { verify(); } + // Defined in ciStreams.hpp + inline Bytecode_lookupswitch(const ciBytecodeStream* stream); void verify() const PRODUCT_RETURN; // Attributes int default_offset() const { return get_Java_u4_at(aligned_offset(1 + 0*jintSize)); } int number_of_pairs() const { return get_Java_u4_at(aligned_offset(1 + 1*jintSize)); } - LookupswitchPair* pair_at(int i) const { assert(0 <= i && i < number_of_pairs(), "pair index out of bounds"); - return (LookupswitchPair*)aligned_addr_at(1 + (1 + i)*2*jintSize); } - // Creation - inline friend Bytecode_lookupswitch* Bytecode_lookupswitch_at(address bcp); + LookupswitchPair pair_at(int i) const { + assert(0 <= i && i < number_of_pairs(), "pair index out of bounds"); + return LookupswitchPair(aligned_addr_at(1 + (1 + i)*2*jintSize)); + } }; -inline Bytecode_lookupswitch* Bytecode_lookupswitch_at(address bcp) { - Bytecode_lookupswitch* b = (Bytecode_lookupswitch*)bcp; - DEBUG_ONLY(b->verify()); - return b; -} - - class Bytecode_tableswitch: public Bytecode { public: + Bytecode_tableswitch(methodOop method, address bcp): Bytecode(method, bcp) { verify(); } + // Defined in ciStreams.hpp + inline Bytecode_tableswitch(const ciBytecodeStream* stream); void verify() const PRODUCT_RETURN; // Attributes @@ -187,52 +178,36 @@ int high_key() const { return get_Java_u4_at(aligned_offset(1 + 2*jintSize)); } int dest_offset_at(int i) const; int length() { return high_key()-low_key()+1; } - - // Creation - inline friend Bytecode_tableswitch* Bytecode_tableswitch_at(address bcp); }; -inline Bytecode_tableswitch* Bytecode_tableswitch_at(address bcp) { - Bytecode_tableswitch* b = (Bytecode_tableswitch*)bcp; - DEBUG_ONLY(b->verify()); - return b; -} - - // Common code for decoding invokes and field references. -class Bytecode_member_ref: public ResourceObj { +class Bytecode_member_ref: public Bytecode { protected: - methodHandle _method; // method containing the bytecode - int _bci; // position of the bytecode + const methodHandle _method; // method containing the bytecode - Bytecode_member_ref(methodHandle method, int bci) : _method(method), _bci(bci) {} + Bytecode_member_ref(methodHandle method, int bci) : Bytecode(method(), method()->bcp_from(bci)), _method(method) {} + + methodHandle method() const { return _method; } public: - // Attributes - methodHandle method() const { return _method; } - int bci() const { return _bci; } - address bcp() const { return _method->bcp_from(bci()); } - Bytecode* bytecode() const { return Bytecode_at(bcp()); } - int index() const; // cache index (loaded from instruction) int pool_index() const; // constant pool index symbolOop name() const; // returns the name of the method or field symbolOop signature() const; // returns the signature of the method or field BasicType result_type(Thread* thread) const; // returns the result type of the getfield or invoke - - Bytecodes::Code code() const { return Bytecodes::code_at(bcp(), _method()); } - Bytecodes::Code java_code() const { return Bytecodes::java_code(code()); } }; // Abstraction for invoke_{virtual, static, interface, special} class Bytecode_invoke: public Bytecode_member_ref { protected: - Bytecode_invoke(methodHandle method, int bci) : Bytecode_member_ref(method, bci) {} + // Constructor that skips verification + Bytecode_invoke(methodHandle method, int bci, bool unused) : Bytecode_member_ref(method, bci) {} public: + Bytecode_invoke(methodHandle method, int bci) : Bytecode_member_ref(method, bci) { verify(); } void verify() const; // Attributes @@ -253,31 +228,20 @@ is_invokespecial() || is_invokedynamic(); } - // Creation - inline friend Bytecode_invoke* Bytecode_invoke_at(methodHandle method, int bci); - - // Like Bytecode_invoke_at. Instead it returns NULL if the bci is not at an invoke. - inline friend Bytecode_invoke* Bytecode_invoke_at_check(methodHandle method, int bci); + // Helper to skip verification. Used is_valid() to check if the result is really an invoke + inline friend Bytecode_invoke Bytecode_invoke_check(methodHandle method, int bci); }; -inline Bytecode_invoke* Bytecode_invoke_at(methodHandle method, int bci) { - Bytecode_invoke* b = new Bytecode_invoke(method, bci); - DEBUG_ONLY(b->verify()); - return b; -} - -inline Bytecode_invoke* Bytecode_invoke_at_check(methodHandle method, int bci) { - Bytecode_invoke* b = new Bytecode_invoke(method, bci); - return b->is_valid() ? b : NULL; +inline Bytecode_invoke Bytecode_invoke_check(methodHandle method, int bci) { + return Bytecode_invoke(method, bci, false); } // Abstraction for all field accesses (put/get field/static) class Bytecode_field: public Bytecode_member_ref { - protected: - Bytecode_field(methodHandle method, int bci) : Bytecode_member_ref(method, bci) {} + public: + Bytecode_field(methodHandle method, int bci) : Bytecode_member_ref(method, bci) { verify(); } - public: // Testers bool is_getfield() const { return java_code() == Bytecodes::_getfield; } bool is_putfield() const { return java_code() == Bytecodes::_putfield; } @@ -292,131 +256,64 @@ is_getstatic() || is_putstatic(); } void verify() const; - - // Creation - inline friend Bytecode_field* Bytecode_field_at(methodHandle method, int bci); }; -inline Bytecode_field* Bytecode_field_at(methodHandle method, int bci) { - Bytecode_field* b = new Bytecode_field(method, bci); - DEBUG_ONLY(b->verify()); - return b; -} - - // Abstraction for checkcast - class Bytecode_checkcast: public Bytecode { public: + Bytecode_checkcast(methodOop method, address bcp): Bytecode(method, bcp) { verify(); } void verify() const { assert(Bytecodes::java_code(code()) == Bytecodes::_checkcast, "check checkcast"); } // Returns index long index() const { return get_index_u2(Bytecodes::_checkcast); }; - - // Creation - inline friend Bytecode_checkcast* Bytecode_checkcast_at(address bcp); }; -inline Bytecode_checkcast* Bytecode_checkcast_at(address bcp) { - Bytecode_checkcast* b = (Bytecode_checkcast*)bcp; - DEBUG_ONLY(b->verify()); - return b; -} - - // Abstraction for instanceof - class Bytecode_instanceof: public Bytecode { public: + Bytecode_instanceof(methodOop method, address bcp): Bytecode(method, bcp) { verify(); } void verify() const { assert(code() == Bytecodes::_instanceof, "check instanceof"); } // Returns index long index() const { return get_index_u2(Bytecodes::_instanceof); }; - - // Creation - inline friend Bytecode_instanceof* Bytecode_instanceof_at(address bcp); }; -inline Bytecode_instanceof* Bytecode_instanceof_at(address bcp) { - Bytecode_instanceof* b = (Bytecode_instanceof*)bcp; - DEBUG_ONLY(b->verify()); - return b; -} - - class Bytecode_new: public Bytecode { public: + Bytecode_new(methodOop method, address bcp): Bytecode(method, bcp) { verify(); } void verify() const { assert(java_code() == Bytecodes::_new, "check new"); } // Returns index long index() const { return get_index_u2(Bytecodes::_new); }; - - // Creation - inline friend Bytecode_new* Bytecode_new_at(address bcp); }; -inline Bytecode_new* Bytecode_new_at(address bcp) { - Bytecode_new* b = (Bytecode_new*)bcp; - DEBUG_ONLY(b->verify()); - return b; -} - - class Bytecode_multianewarray: public Bytecode { public: + Bytecode_multianewarray(methodOop method, address bcp): Bytecode(method, bcp) { verify(); } void verify() const { assert(java_code() == Bytecodes::_multianewarray, "check new"); } // Returns index long index() const { return get_index_u2(Bytecodes::_multianewarray); }; - - // Creation - inline friend Bytecode_multianewarray* Bytecode_multianewarray_at(address bcp); }; -inline Bytecode_multianewarray* Bytecode_multianewarray_at(address bcp) { - Bytecode_multianewarray* b = (Bytecode_multianewarray*)bcp; - DEBUG_ONLY(b->verify()); - return b; -} - - class Bytecode_anewarray: public Bytecode { public: + Bytecode_anewarray(methodOop method, address bcp): Bytecode(method, bcp) { verify(); } void verify() const { assert(java_code() == Bytecodes::_anewarray, "check anewarray"); } // Returns index long index() const { return get_index_u2(Bytecodes::_anewarray); }; - - // Creation - inline friend Bytecode_anewarray* Bytecode_anewarray_at(address bcp); }; -inline Bytecode_anewarray* Bytecode_anewarray_at(address bcp) { - Bytecode_anewarray* b = (Bytecode_anewarray*)bcp; - DEBUG_ONLY(b->verify()); - return b; -} - - // Abstraction for ldc, ldc_w and ldc2_w - -class Bytecode_loadconstant: public ResourceObj { +class Bytecode_loadconstant: public Bytecode { private: - int _bci; - methodHandle _method; - - Bytecodes::Code code() const { return bytecode()->code(); } + const methodHandle _method; int raw_index() const; - Bytecode_loadconstant(methodHandle method, int bci) : _method(method), _bci(bci) {} - public: - // Attributes - methodHandle method() const { return _method; } - int bci() const { return _bci; } - address bcp() const { return _method->bcp_from(bci()); } - Bytecode* bytecode() const { return Bytecode_at(bcp()); } + Bytecode_loadconstant(methodHandle method, int bci): Bytecode(method(), method->bcp_from(bci)), _method(method) { verify(); } void verify() const { assert(_method.not_null(), "must supply method"); @@ -437,15 +334,6 @@ BasicType result_type() const; // returns the result type of the ldc oop resolve_constant(TRAPS) const; - - // Creation - inline friend Bytecode_loadconstant* Bytecode_loadconstant_at(methodHandle method, int bci); }; -inline Bytecode_loadconstant* Bytecode_loadconstant_at(methodHandle method, int bci) { - Bytecode_loadconstant* b = new Bytecode_loadconstant(method, bci); - DEBUG_ONLY(b->verify()); - return b; -} - #endif // SHARE_VM_INTERPRETER_BYTECODE_HPP diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/interpreter/bytecodeInterpreter.cpp --- a/hotspot/src/share/vm/interpreter/bytecodeInterpreter.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/interpreter/bytecodeInterpreter.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -831,11 +831,11 @@ // much like trying to deopt at a poll return. In that has we simply // get out of here // - if ( Bytecodes::code_at(pc, METHOD) == Bytecodes::_return_register_finalizer) { + if ( Bytecodes::code_at(METHOD, pc) == Bytecodes::_return_register_finalizer) { // this will do the right thing even if an exception is pending. goto handle_return; } - UPDATE_PC(Bytecodes::length_at(pc)); + UPDATE_PC(Bytecodes::length_at(METHOD, pc)); if (THREAD->has_pending_exception()) goto handle_exception; goto run; } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/interpreter/bytecodeStream.cpp --- a/hotspot/src/share/vm/interpreter/bytecodeStream.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/interpreter/bytecodeStream.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -59,7 +59,7 @@ // in raw mode, pretend indy is "bJJ__" assert(size == 2, "raw invokedynamic instruction has 2-byte index only"); } else { - bytecode()->assert_index_size(size, raw_code(), is_wide()); + bytecode().assert_index_size(size, raw_code(), is_wide()); } } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/interpreter/bytecodeStream.hpp --- a/hotspot/src/share/vm/interpreter/bytecodeStream.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/interpreter/bytecodeStream.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -105,14 +105,14 @@ bool is_last_bytecode() const { return _next_bci >= _end_bci; } address bcp() const { return method()->code_base() + _bci; } - Bytecode* bytecode() const { return Bytecode_at(bcp()); } + Bytecode bytecode() const { return Bytecode(_method(), bcp()); } // State changes void set_next_bci(int bci) { assert(0 <= bci && bci <= method()->code_size(), "illegal bci"); _next_bci = bci; } // Bytecode-specific attributes - int dest() const { return bci() + bytecode()->get_offset_s2(raw_code()); } - int dest_w() const { return bci() + bytecode()->get_offset_s4(raw_code()); } + int dest() const { return bci() + bytecode().get_offset_s2(raw_code()); } + int dest_w() const { return bci() + bytecode().get_offset_s4(raw_code()); } // One-byte indices. int get_index_u1() const { assert_raw_index_size(1); return *(jubyte*)(bcp()+1); } @@ -189,7 +189,7 @@ } else { // get bytecode address bcp = this->bcp(); - raw_code = Bytecodes::code_at(bcp); + raw_code = Bytecodes::code_at(_method(), bcp); code = Bytecodes::java_code(raw_code); // set next bytecode position // @@ -197,7 +197,7 @@ // tty bytecode otherwise the stepping is wrong! // (carefull: length_for(...) must be used first!) int l = Bytecodes::length_for(code); - if (l == 0) l = Bytecodes::length_at(bcp); + if (l == 0) l = Bytecodes::length_at(_method(), bcp); _next_bci += l; assert(_bci < _next_bci, "length must be > 0"); // set attributes @@ -219,16 +219,16 @@ Bytecodes::Code code() const { return _code; } // Unsigned indices, widening - int get_index() const { return is_wide() ? bytecode()->get_index_u2(raw_code(), true) : get_index_u1(); } + int get_index() const { return is_wide() ? bytecode().get_index_u2(raw_code(), true) : get_index_u1(); } // Get an unsigned 2-byte index, swapping the bytes if necessary. int get_index_u2() const { assert_raw_stream(false); - return bytecode()->get_index_u2(raw_code(), false); } + return bytecode().get_index_u2(raw_code(), false); } // Get an unsigned 2-byte index in native order. int get_index_u2_cpcache() const { assert_raw_stream(false); - return bytecode()->get_index_u2_cpcache(raw_code()); } + return bytecode().get_index_u2_cpcache(raw_code()); } int get_index_u4() const { assert_raw_stream(false); - return bytecode()->get_index_u4(raw_code()); } - bool has_index_u4() const { return bytecode()->has_index_u4(raw_code()); } + return bytecode().get_index_u4(raw_code()); } + bool has_index_u4() const { return bytecode().has_index_u4(raw_code()); } }; #endif // SHARE_VM_INTERPRETER_BYTECODESTREAM_HPP diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/interpreter/bytecodeTracer.cpp --- a/hotspot/src/share/vm/interpreter/bytecodeTracer.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/interpreter/bytecodeTracer.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -100,9 +100,9 @@ Bytecodes::Code code; if (is_wide()) { // bcp wasn't advanced if previous bytecode was _wide. - code = Bytecodes::code_at(bcp+1); + code = Bytecodes::code_at(method(), bcp+1); } else { - code = Bytecodes::code_at(bcp); + code = Bytecodes::code_at(method(), bcp); } _code = code; int bci = bcp - method->code_base(); @@ -127,11 +127,11 @@ void trace(methodHandle method, address bcp, outputStream* st) { _current_method = method(); ResourceMark rm; - Bytecodes::Code code = Bytecodes::code_at(bcp); + Bytecodes::Code code = Bytecodes::code_at(method(), bcp); // Set is_wide _is_wide = (code == Bytecodes::_wide); if (is_wide()) { - code = Bytecodes::code_at(bcp+1); + code = Bytecodes::code_at(method(), bcp+1); } _code = code; int bci = bcp - method->code_base(); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/interpreter/bytecodes.cpp --- a/hotspot/src/share/vm/interpreter/bytecodes.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/interpreter/bytecodes.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -54,18 +54,46 @@ Bytecodes::Code Bytecodes::_java_code [Bytecodes::number_of_codes]; u_short Bytecodes::_flags [(1<contains(bcp); +} +#endif + +bool Bytecodes::check_must_rewrite(Bytecodes::Code code) { + assert(can_rewrite(code), "post-check only"); + + // Some codes are conditionally rewriting. Look closely at them. + switch (code) { + case Bytecodes::_aload_0: + // Even if RewriteFrequentPairs is turned on, + // the _aload_0 code might delay its rewrite until + // a following _getfield rewrites itself. + return false; + + case Bytecodes::_lookupswitch: + return false; // the rewrite is not done by the interpreter + + case Bytecodes::_new: + // (Could actually look at the class here, but the profit would be small.) + return false; // the rewrite is not always done + } + + // No other special cases. + return true; +} Bytecodes::Code Bytecodes::code_at(methodOop method, int bci) { - return code_at(method->bcp_from(bci), method); + return code_at(method, method->bcp_from(bci)); } -Bytecodes::Code Bytecodes::non_breakpoint_code_at(address bcp, methodOop method) { - if (method == NULL) method = methodOopDesc::method_from_bcp(bcp); +Bytecodes::Code Bytecodes::non_breakpoint_code_at(const methodOopDesc* method, address bcp) { + assert(method != NULL, "must have the method for breakpoint conversion"); + assert(method->contains(bcp), "must be valid bcp in method"); return method->orig_bytecode_at(method->bci_from(bcp)); } -int Bytecodes::special_length_at(address bcp, address end) { - Code code = code_at(bcp); +int Bytecodes::special_length_at(Bytecodes::Code code, address bcp, address end) { switch (code) { case _wide: if (end != NULL && bcp + 1 >= end) { @@ -120,7 +148,7 @@ if (code == _breakpoint) { return 1; } else { - return special_length_at(bcp, end); + return special_length_at(code, bcp, end); } } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/interpreter/bytecodes.hpp --- a/hotspot/src/share/vm/interpreter/bytecodes.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/interpreter/bytecodes.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -342,6 +342,12 @@ static void pd_initialize(); // platform specific initialization static Code pd_base_code_for(Code code); // platform specific base_code_for implementation + // Verify that bcp points into method +#ifdef ASSERT + static bool check_method(const methodOopDesc* method, address bcp); +#endif + static bool check_must_rewrite(Bytecodes::Code bc); + public: // Conversion static void check (Code code) { assert(is_defined(code), "illegal code"); } @@ -349,22 +355,30 @@ static Code cast (int code) { return (Code)code; } - // Fetch a bytecode, hiding breakpoints as necessary: - static Code code_at(address bcp, methodOop method = NULL) { - Code code = cast(*bcp); return (code != _breakpoint) ? code : non_breakpoint_code_at(bcp, method); - } - static Code java_code_at(address bcp, methodOop method = NULL) { - return java_code(code_at(bcp, method)); - } + // Fetch a bytecode, hiding breakpoints as necessary. The method + // argument is used for conversion of breakpoints into the original + // bytecode. The CI uses these methods but guarantees that + // breakpoints are hidden so the method argument should be passed as + // NULL since in that case the bcp and methodOop are unrelated + // memory. + static Code code_at(const methodOopDesc* method, address bcp) { + assert(method == NULL || check_method(method, bcp), "bcp must point into method"); + Code code = cast(*bcp); + assert(code != _breakpoint || method != NULL, "need methodOop to decode breakpoint"); + return (code != _breakpoint) ? code : non_breakpoint_code_at(method, bcp); + } + static Code java_code_at(const methodOopDesc* method, address bcp) { + return java_code(code_at(method, bcp)); + } - // Fetch a bytecode or a breakpoint: - static Code code_or_bp_at(address bcp) { return (Code)cast(*bcp); } + // Fetch a bytecode or a breakpoint: + static Code code_or_bp_at(address bcp) { return (Code)cast(*bcp); } - static Code code_at(methodOop method, int bci); - static bool is_active_breakpoint_at(address bcp) { return (Code)*bcp == _breakpoint; } + static Code code_at(methodOop method, int bci); + static bool is_active_breakpoint_at(address bcp) { return (Code)*bcp == _breakpoint; } - // find a bytecode, behind a breakpoint if necessary: - static Code non_breakpoint_code_at(address bcp, methodOop method = NULL); + // find a bytecode, behind a breakpoint if necessary: + static Code non_breakpoint_code_at(const methodOopDesc* method, address bcp); // Bytecode attributes static bool is_defined (int code) { return 0 <= code && code < number_of_codes && flags(code, false) != 0; } @@ -379,14 +393,17 @@ static bool can_trap (Code code) { check(code); return has_all_flags(code, _bc_can_trap, false); } static Code java_code (Code code) { check(code); return _java_code [code]; } static bool can_rewrite (Code code) { check(code); return has_all_flags(code, _bc_can_rewrite, false); } + static bool must_rewrite(Bytecodes::Code code) { return can_rewrite(code) && check_must_rewrite(code); } static bool native_byte_order(Code code) { check(code); return has_all_flags(code, _fmt_has_nbo, false); } static bool uses_cp_cache (Code code) { check(code); return has_all_flags(code, _fmt_has_j, false); } // if 'end' is provided, it indicates the end of the code buffer which // should not be read past when parsing. - static int special_length_at(address bcp, address end = NULL); + static int special_length_at(Bytecodes::Code code, address bcp, address end = NULL); + static int special_length_at(methodOop method, address bcp, address end = NULL) { return special_length_at(code_at(method, bcp), bcp, end); } static int raw_special_length_at(address bcp, address end = NULL); - static int length_at (address bcp) { int l = length_for(code_at(bcp)); return l > 0 ? l : special_length_at(bcp); } - static int java_length_at (address bcp) { int l = length_for(java_code_at(bcp)); return l > 0 ? l : special_length_at(bcp); } + static int length_for_code_at(Bytecodes::Code code, address bcp) { int l = length_for(code); return l > 0 ? l : special_length_at(code, bcp); } + static int length_at (methodOop method, address bcp) { return length_for_code_at(code_at(method, bcp), bcp); } + static int java_length_at (methodOop method, address bcp) { return length_for_code_at(java_code_at(method, bcp), bcp); } static bool is_java_code (Code code) { return 0 <= code && code < number_of_java_codes; } static bool is_aload (Code code) { return (code == _aload || code == _aload_0 || code == _aload_1 diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/interpreter/interpreter.cpp --- a/hotspot/src/share/vm/interpreter/interpreter.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/interpreter/interpreter.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -237,10 +237,9 @@ // Return true if the interpreter can prove that the given bytecode has // not yet been executed (in Java semantics, not in actual operation). bool AbstractInterpreter::is_not_reached(methodHandle method, int bci) { - address bcp = method->bcp_from(bci); - Bytecodes::Code code = Bytecodes::code_at(bcp, method()); + Bytecodes::Code code = method()->code_at(bci); - if (!Bytecode_at(bcp)->must_rewrite(code)) { + if (!Bytecodes::must_rewrite(code)) { // might have been reached return false; } @@ -286,12 +285,12 @@ // If deoptimization happens, this function returns the point of next bytecode to continue execution address AbstractInterpreter::deopt_continue_after_entry(methodOop method, address bcp, int callee_parameters, bool is_top_frame) { assert(method->contains(bcp), "just checkin'"); - Bytecodes::Code code = Bytecodes::java_code_at(bcp); + Bytecodes::Code code = Bytecodes::java_code_at(method, bcp); assert(!Interpreter::bytecode_should_reexecute(code), "should not reexecute"); int bci = method->bci_from(bcp); int length = -1; // initial value for debugging // compute continuation length - length = Bytecodes::length_at(bcp); + length = Bytecodes::length_at(method, bcp); // compute result type BasicType type = T_ILLEGAL; @@ -303,7 +302,7 @@ Thread *thread = Thread::current(); ResourceMark rm(thread); methodHandle mh(thread, method); - type = Bytecode_invoke_at(mh, bci)->result_type(thread); + type = Bytecode_invoke(mh, bci).result_type(thread); // since the cache entry might not be initialized: // (NOT needed for the old calling convension) if (!is_top_frame) { @@ -317,7 +316,7 @@ Thread *thread = Thread::current(); ResourceMark rm(thread); methodHandle mh(thread, method); - type = Bytecode_invoke_at(mh, bci)->result_type(thread); + type = Bytecode_invoke(mh, bci).result_type(thread); // since the cache entry might not be initialized: // (NOT needed for the old calling convension) if (!is_top_frame) { @@ -334,7 +333,7 @@ Thread *thread = Thread::current(); ResourceMark rm(thread); methodHandle mh(thread, method); - type = Bytecode_loadconstant_at(mh, bci)->result_type(); + type = Bytecode_loadconstant(mh, bci).result_type(); break; } @@ -356,7 +355,7 @@ // Interpreter::deopt_entry(vtos, 0) like others address AbstractInterpreter::deopt_reexecute_entry(methodOop method, address bcp) { assert(method->contains(bcp), "just checkin'"); - Bytecodes::Code code = Bytecodes::java_code_at(bcp); + Bytecodes::Code code = Bytecodes::java_code_at(method, bcp); #ifdef COMPILER1 if(code == Bytecodes::_athrow ) { return Interpreter::rethrow_exception_entry(); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/interpreter/interpreterRuntime.cpp --- a/hotspot/src/share/vm/interpreter/interpreterRuntime.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/interpreter/interpreterRuntime.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -132,9 +132,9 @@ bytecode == Bytecodes::_fast_aldc_w, "wrong bc"); ResourceMark rm(thread); methodHandle m (thread, method(thread)); - Bytecode_loadconstant* ldc = Bytecode_loadconstant_at(m, bci(thread)); - oop result = ldc->resolve_constant(THREAD); - DEBUG_ONLY(ConstantPoolCacheEntry* cpce = m->constants()->cache()->entry_at(ldc->cache_index())); + Bytecode_loadconstant ldc(m, bci(thread)); + oop result = ldc.resolve_constant(THREAD); + DEBUG_ONLY(ConstantPoolCacheEntry* cpce = m->constants()->cache()->entry_at(ldc.cache_index())); assert(result == cpce->f1(), "expected result for assembly code"); } IRT_END @@ -672,8 +672,8 @@ if (bytecode == Bytecodes::_invokevirtual || bytecode == Bytecodes::_invokeinterface) { ResourceMark rm(thread); methodHandle m (thread, method(thread)); - Bytecode_invoke* call = Bytecode_invoke_at(m, bci(thread)); - symbolHandle signature (thread, call->signature()); + Bytecode_invoke call(m, bci(thread)); + symbolHandle signature (thread, call.signature()); receiver = Handle(thread, thread->last_frame().interpreter_callee_receiver(signature)); assert(Universe::heap()->is_in_reserved_or_null(receiver()), @@ -756,7 +756,7 @@ caller_bci = caller_method->bci_from(caller_bcp); site_index = Bytes::get_native_u4(caller_bcp+1); } - assert(site_index == InterpreterRuntime::bytecode(thread)->get_index_u4(bytecode), ""); + assert(site_index == InterpreterRuntime::bytecode(thread).get_index_u4(bytecode), ""); assert(constantPoolCacheOopDesc::is_secondary_index(site_index), "proper format"); // there is a second CPC entries that is of interest; it caches signature info: int main_index = pool->cache()->secondary_entry_at(site_index)->main_entry_index(); @@ -884,7 +884,7 @@ return mdo->bci_to_di(bci); IRT_END -IRT_ENTRY(jint, InterpreterRuntime::profile_method(JavaThread* thread, address cur_bcp)) +IRT_ENTRY(void, InterpreterRuntime::profile_method(JavaThread* thread)) // use UnlockFlagSaver to clear and restore the _do_not_unlock_if_synchronized // flag, in case this method triggers classloading which will call into Java. UnlockFlagSaver fs(thread); @@ -893,16 +893,12 @@ frame fr = thread->last_frame(); assert(fr.is_interpreted_frame(), "must come from interpreter"); methodHandle method(thread, fr.interpreter_frame_method()); - int bci = method->bci_from(cur_bcp); methodOopDesc::build_interpreter_method_data(method, THREAD); if (HAS_PENDING_EXCEPTION) { assert((PENDING_EXCEPTION->is_a(SystemDictionary::OutOfMemoryError_klass())), "we expect only an OOM error here"); CLEAR_PENDING_EXCEPTION; // and fall through... } - methodDataOop mdo = method->method_data(); - if (mdo == NULL) return 0; - return mdo->bci_to_di(bci); IRT_END @@ -1245,9 +1241,9 @@ assert(fr.is_interpreted_frame(), ""); jint bci = fr.interpreter_frame_bci(); methodHandle mh(thread, fr.interpreter_frame_method()); - Bytecode_invoke* invoke = Bytecode_invoke_at(mh, bci); - ArgumentSizeComputer asc(invoke->signature()); - int size_of_arguments = (asc.size() + (invoke->has_receiver() ? 1 : 0)); // receiver + Bytecode_invoke invoke(mh, bci); + ArgumentSizeComputer asc(invoke.signature()); + int size_of_arguments = (asc.size() + (invoke.has_receiver() ? 1 : 0)); // receiver Copy::conjoint_jbytes(src_address, dest_address, size_of_arguments * Interpreter::stackElementSize); IRT_END diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/interpreter/interpreterRuntime.hpp --- a/hotspot/src/share/vm/interpreter/interpreterRuntime.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/interpreter/interpreterRuntime.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -58,16 +58,16 @@ static void set_bcp_and_mdp(address bcp, JavaThread*thread); static Bytecodes::Code code(JavaThread *thread) { // pass method to avoid calling unsafe bcp_to_method (partial fix 4926272) - return Bytecodes::code_at(bcp(thread), method(thread)); + return Bytecodes::code_at(method(thread), bcp(thread)); } static bool already_resolved(JavaThread *thread) { return cache_entry(thread)->is_resolved(code(thread)); } - static Bytecode* bytecode(JavaThread *thread) { return Bytecode_at(bcp(thread)); } + static Bytecode bytecode(JavaThread *thread) { return Bytecode(method(thread), bcp(thread)); } static int get_index_u1(JavaThread *thread, Bytecodes::Code bc) - { return bytecode(thread)->get_index_u1(bc); } + { return bytecode(thread).get_index_u1(bc); } static int get_index_u2(JavaThread *thread, Bytecodes::Code bc) - { return bytecode(thread)->get_index_u2(bc); } + { return bytecode(thread).get_index_u2(bc); } static int get_index_u2_cpcache(JavaThread *thread, Bytecodes::Code bc) - { return bytecode(thread)->get_index_u2_cpcache(bc); } + { return bytecode(thread).get_index_u2_cpcache(bc); } static int number_of_dimensions(JavaThread *thread) { return bcp(thread)[3]; } static ConstantPoolCacheEntry* cache_entry_at(JavaThread *thread, int i) { return method(thread)->constants()->cache()->entry_at(i); } @@ -164,7 +164,7 @@ // Interpreter profiling support static jint bcp_to_di(methodOopDesc* method, address cur_bcp); - static jint profile_method(JavaThread* thread, address cur_bcp); + static void profile_method(JavaThread* thread); static void update_mdp_for_ret(JavaThread* thread, int bci); #ifdef ASSERT static void verify_mdp(methodOopDesc* method, address bcp, address mdp); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/interpreter/rewriter.cpp --- a/hotspot/src/share/vm/interpreter/rewriter.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/interpreter/rewriter.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -221,7 +221,7 @@ // call to calculate the length. bc_length = Bytecodes::length_for(c); if (bc_length == 0) { - bc_length = Bytecodes::length_at(bcp); + bc_length = Bytecodes::length_at(method, bcp); // length_at will put us at the bytecode after the one modified // by 'wide'. We don't currently examine any of the bytecodes @@ -237,9 +237,9 @@ switch (c) { case Bytecodes::_lookupswitch : { #ifndef CC_INTERP - Bytecode_lookupswitch* bc = Bytecode_lookupswitch_at(bcp); + Bytecode_lookupswitch bc(method, bcp); (*bcp) = ( - bc->number_of_pairs() < BinarySwitchThreshold + bc.number_of_pairs() < BinarySwitchThreshold ? Bytecodes::_fast_linearswitch : Bytecodes::_fast_binaryswitch ); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/interpreter/templateInterpreter.cpp --- a/hotspot/src/share/vm/interpreter/templateInterpreter.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/interpreter/templateInterpreter.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -592,7 +592,7 @@ // that do not return "Interpreter::deopt_entry(vtos, 0)" address TemplateInterpreter::deopt_reexecute_entry(methodOop method, address bcp) { assert(method->contains(bcp), "just checkin'"); - Bytecodes::Code code = Bytecodes::java_code_at(bcp); + Bytecodes::Code code = Bytecodes::java_code_at(method, bcp); if (code == Bytecodes::_return) { // This is used for deopt during registration of finalizers // during Object.. We simply need to resume execution at diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/memory/genMarkSweep.cpp --- a/hotspot/src/share/vm/memory/genMarkSweep.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/memory/genMarkSweep.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -158,6 +158,7 @@ Threads::gc_epilogue(); CodeCache::gc_epilogue(); + JvmtiExport::gc_epilogue(); if (PrintGC && !PrintGCDetails) { gch->print_heap_change(gch_prev_used); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/oops/arrayKlass.hpp --- a/hotspot/src/share/vm/oops/arrayKlass.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/oops/arrayKlass.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -36,8 +36,8 @@ friend class VMStructs; private: int _dimension; // This is n'th-dimensional array. - klassOop _higher_dimension; // Refers the (n+1)'th-dimensional array (if present). - klassOop _lower_dimension; // Refers the (n-1)'th-dimensional array (if present). + volatile klassOop _higher_dimension; // Refers the (n+1)'th-dimensional array (if present). + volatile klassOop _lower_dimension; // Refers the (n-1)'th-dimensional array (if present). int _vtable_len; // size of vtable for this klass juint _alloc_size; // allocation profiling support oop _component_mirror; // component type, as a java/lang/Class diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/oops/generateOopMap.cpp --- a/hotspot/src/share/vm/oops/generateOopMap.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/oops/generateOopMap.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -535,23 +535,23 @@ (*jmpFct)(this, bcs->dest_w(), data); break; case Bytecodes::_tableswitch: - { Bytecode_tableswitch *tableswitch = Bytecode_tableswitch_at(bcs->bcp()); - int len = tableswitch->length(); + { Bytecode_tableswitch tableswitch(method(), bcs->bcp()); + int len = tableswitch.length(); - (*jmpFct)(this, bci + tableswitch->default_offset(), data); /* Default. jump address */ + (*jmpFct)(this, bci + tableswitch.default_offset(), data); /* Default. jump address */ while (--len >= 0) { - (*jmpFct)(this, bci + tableswitch->dest_offset_at(len), data); + (*jmpFct)(this, bci + tableswitch.dest_offset_at(len), data); } break; } case Bytecodes::_lookupswitch: - { Bytecode_lookupswitch *lookupswitch = Bytecode_lookupswitch_at(bcs->bcp()); - int npairs = lookupswitch->number_of_pairs(); - (*jmpFct)(this, bci + lookupswitch->default_offset(), data); /* Default. */ + { Bytecode_lookupswitch lookupswitch(method(), bcs->bcp()); + int npairs = lookupswitch.number_of_pairs(); + (*jmpFct)(this, bci + lookupswitch.default_offset(), data); /* Default. */ while(--npairs >= 0) { - LookupswitchPair *pair = lookupswitch->pair_at(npairs); - (*jmpFct)(this, bci + pair->offset(), data); + LookupswitchPair pair = lookupswitch.pair_at(npairs); + (*jmpFct)(this, bci + pair.offset(), data); } break; } @@ -977,7 +977,7 @@ #ifdef ASSERT if (blockNum + 1 < bbNo) { address bcp = _method->bcp_from(bb->_end_bci); - int bc_len = Bytecodes::java_length_at(bcp); + int bc_len = Bytecodes::java_length_at(_method(), bcp); assert(bb->_end_bci + bc_len == bb[1]._bci, "unmatched bci info in basicblock"); } #endif @@ -985,7 +985,7 @@ #ifdef ASSERT { BasicBlock *bb = &_basic_blocks[bbNo-1]; address bcp = _method->bcp_from(bb->_end_bci); - int bc_len = Bytecodes::java_length_at(bcp); + int bc_len = Bytecodes::java_length_at(_method(), bcp); assert(bb->_end_bci + bc_len == _method->code_size(), "wrong end bci"); } #endif @@ -1837,14 +1837,14 @@ void GenerateOopMap::do_ldc(int bci) { - Bytecode_loadconstant* ldc = Bytecode_loadconstant_at(method(), bci); + Bytecode_loadconstant ldc(method(), bci); constantPoolOop cp = method()->constants(); - BasicType bt = ldc->result_type(); + BasicType bt = ldc.result_type(); CellTypeState cts = (bt == T_OBJECT) ? CellTypeState::make_line_ref(bci) : valCTS; // Make sure bt==T_OBJECT is the same as old code (is_pointer_entry). // Note that CONSTANT_MethodHandle entries are u2 index pairs, not pointer-entries, // and they are processed by _fast_aldc and the CP cache. - assert((ldc->has_cache_index() || cp->is_pointer_entry(ldc->pool_index())) + assert((ldc.has_cache_index() || cp->is_pointer_entry(ldc.pool_index())) ? (bt == T_OBJECT) : true, "expected object type"); ppush1(cts); } @@ -2343,7 +2343,7 @@ bool GenerateOopMap::rewrite_load_or_store(BytecodeStream *bcs, Bytecodes::Code bcN, Bytecodes::Code bc0, unsigned int varNo) { assert(bcN == Bytecodes::_astore || bcN == Bytecodes::_aload, "wrong argument (bcN)"); assert(bc0 == Bytecodes::_astore_0 || bc0 == Bytecodes::_aload_0, "wrong argument (bc0)"); - int ilen = Bytecodes::length_at(bcs->bcp()); + int ilen = Bytecodes::length_at(_method(), bcs->bcp()); int newIlen; if (ilen == 4) { diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/oops/methodDataOop.cpp --- a/hotspot/src/share/vm/oops/methodDataOop.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/oops/methodDataOop.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -417,11 +417,11 @@ int MultiBranchData::compute_cell_count(BytecodeStream* stream) { int cell_count = 0; if (stream->code() == Bytecodes::_tableswitch) { - Bytecode_tableswitch* sw = Bytecode_tableswitch_at(stream->bcp()); - cell_count = 1 + per_case_cell_count * (1 + sw->length()); // 1 for default + Bytecode_tableswitch sw(stream->method()(), stream->bcp()); + cell_count = 1 + per_case_cell_count * (1 + sw.length()); // 1 for default } else { - Bytecode_lookupswitch* sw = Bytecode_lookupswitch_at(stream->bcp()); - cell_count = 1 + per_case_cell_count * (sw->number_of_pairs() + 1); // 1 for default + Bytecode_lookupswitch sw(stream->method()(), stream->bcp()); + cell_count = 1 + per_case_cell_count * (sw.number_of_pairs() + 1); // 1 for default } return cell_count; } @@ -434,35 +434,35 @@ int target_di; int offset; if (stream->code() == Bytecodes::_tableswitch) { - Bytecode_tableswitch* sw = Bytecode_tableswitch_at(stream->bcp()); - int len = sw->length(); + Bytecode_tableswitch sw(stream->method()(), stream->bcp()); + int len = sw.length(); assert(array_len() == per_case_cell_count * (len + 1), "wrong len"); for (int count = 0; count < len; count++) { - target = sw->dest_offset_at(count) + bci(); + target = sw.dest_offset_at(count) + bci(); my_di = mdo->dp_to_di(dp()); target_di = mdo->bci_to_di(target); offset = target_di - my_di; set_displacement_at(count, offset); } - target = sw->default_offset() + bci(); + target = sw.default_offset() + bci(); my_di = mdo->dp_to_di(dp()); target_di = mdo->bci_to_di(target); offset = target_di - my_di; set_default_displacement(offset); } else { - Bytecode_lookupswitch* sw = Bytecode_lookupswitch_at(stream->bcp()); - int npairs = sw->number_of_pairs(); + Bytecode_lookupswitch sw(stream->method()(), stream->bcp()); + int npairs = sw.number_of_pairs(); assert(array_len() == per_case_cell_count * (npairs + 1), "wrong len"); for (int count = 0; count < npairs; count++) { - LookupswitchPair *pair = sw->pair_at(count); - target = pair->offset() + bci(); + LookupswitchPair pair = sw.pair_at(count); + target = pair.offset() + bci(); my_di = mdo->dp_to_di(dp()); target_di = mdo->bci_to_di(target); offset = target_di - my_di; set_displacement_at(count, offset); } - target = sw->default_offset() + bci(); + target = sw.default_offset() + bci(); my_di = mdo->dp_to_di(dp()); target_di = mdo->bci_to_di(target); offset = target_di - my_di; diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/oops/methodOop.cpp --- a/hotspot/src/share/vm/oops/methodOop.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/oops/methodOop.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -150,17 +150,6 @@ return -1; } -methodOop methodOopDesc::method_from_bcp(address bcp) { - debug_only(static int count = 0; count++); - assert(Universe::heap()->is_in_permanent(bcp), "bcp not in perm_gen"); - // TO DO: this may be unsafe in some configurations - HeapWord* p = Universe::heap()->block_start(bcp); - assert(Universe::heap()->block_is_obj(p), "must be obj"); - assert(oop(p)->is_constMethod(), "not a method"); - return constMethodOop(p)->method(); -} - - void methodOopDesc::mask_for(int bci, InterpreterOopMap* mask) { Thread* myThread = Thread::current(); @@ -469,11 +458,10 @@ bool methodOopDesc::is_accessor() const { if (code_size() != 5) return false; if (size_of_parameters() != 1) return false; - methodOop m = (methodOop)this; // pass to code_at() to avoid method_from_bcp - if (Bytecodes::java_code_at(code_base()+0, m) != Bytecodes::_aload_0 ) return false; - if (Bytecodes::java_code_at(code_base()+1, m) != Bytecodes::_getfield) return false; - if (Bytecodes::java_code_at(code_base()+4, m) != Bytecodes::_areturn && - Bytecodes::java_code_at(code_base()+4, m) != Bytecodes::_ireturn ) return false; + if (java_code_at(0) != Bytecodes::_aload_0 ) return false; + if (java_code_at(1) != Bytecodes::_getfield) return false; + if (java_code_at(4) != Bytecodes::_areturn && + java_code_at(4) != Bytecodes::_ireturn ) return false; return true; } @@ -1414,7 +1402,7 @@ } -Bytecodes::Code methodOopDesc::orig_bytecode_at(int bci) { +Bytecodes::Code methodOopDesc::orig_bytecode_at(int bci) const { BreakpointInfo* bp = instanceKlass::cast(method_holder())->breakpoints(); for (; bp != NULL; bp = bp->next()) { if (bp->match(this, bci)) { diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/oops/methodOop.hpp --- a/hotspot/src/share/vm/oops/methodOop.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/oops/methodOop.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -196,8 +196,15 @@ static char* name_and_sig_as_C_string(Klass* klass, symbolOop method_name, symbolOop signature); static char* name_and_sig_as_C_string(Klass* klass, symbolOop method_name, symbolOop signature, char* buf, int size); + Bytecodes::Code java_code_at(int bci) const { + return Bytecodes::java_code_at(this, bcp_from(bci)); + } + Bytecodes::Code code_at(int bci) const { + return Bytecodes::code_at(this, bcp_from(bci)); + } + // JVMTI breakpoints - Bytecodes::Code orig_bytecode_at(int bci); + Bytecodes::Code orig_bytecode_at(int bci) const; void set_orig_bytecode_at(int bci, Bytecodes::Code code); void set_breakpoint(int bci); void clear_breakpoint(int bci); @@ -655,8 +662,6 @@ void set_queued_for_compilation() { _access_flags.set_queued_for_compilation(); } void clear_queued_for_compilation() { _access_flags.clear_queued_for_compilation(); } - static methodOop method_from_bcp(address bcp); - // Resolve all classes in signature, return 'true' if successful static bool load_signature_classes(methodHandle m, TRAPS); @@ -787,11 +792,11 @@ void set_next(BreakpointInfo* n) { _next = n; } // helps for searchers - bool match(methodOop m, int bci) { + bool match(const methodOopDesc* m, int bci) { return bci == _bci && match(m); } - bool match(methodOop m) { + bool match(const methodOopDesc* m) { return _name_index == m->name_index() && _signature_index == m->signature_index(); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/oops/objArrayKlass.cpp --- a/hotspot/src/share/vm/oops/objArrayKlass.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/oops/objArrayKlass.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -235,8 +235,9 @@ objArrayKlassKlass::cast(Universe::objArrayKlassKlassObj())-> allocate_objArray_klass(dimension + 1, this_oop, CHECK_NULL); ak = objArrayKlassHandle(THREAD, new_klass); + ak->set_lower_dimension(this_oop()); + OrderAccess::storestore(); this_oop->set_higher_dimension(ak()); - ak->set_lower_dimension(this_oop()); assert(ak->oop_is_objArray(), "incorrect initialization of objArrayKlass"); } } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/oops/typeArrayKlass.cpp --- a/hotspot/src/share/vm/oops/typeArrayKlass.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/oops/typeArrayKlass.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -179,6 +179,7 @@ dimension + 1, h_this, CHECK_NULL); h_ak = objArrayKlassHandle(THREAD, oak); h_ak->set_lower_dimension(h_this()); + OrderAccess::storestore(); h_this->set_higher_dimension(h_ak()); assert(h_ak->oop_is_objArray(), "incorrect initialization of objArrayKlass"); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/opto/type.cpp --- a/hotspot/src/share/vm/opto/type.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/opto/type.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -2187,8 +2187,11 @@ case TypePtr::NotNull: return this; case TypePtr::Null: - case TypePtr::Constant: - return make( _bits+offset ); + case TypePtr::Constant: { + address bits = _bits+offset; + if ( bits == 0 ) return TypePtr::NULL_PTR; + return make( bits ); + } default: ShouldNotReachHere(); } return NULL; // Lint noise diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/prims/jvm.h --- a/hotspot/src/share/vm/prims/jvm.h Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/prims/jvm.h Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1649,7 +1649,8 @@ * the new bit is also added in the main/baseline. */ unsigned int thread_park_blocker : 1; - unsigned int : 31; + unsigned int post_vm_init_hook_enabled : 1; + unsigned int : 30; unsigned int : 32; unsigned int : 32; } jdk_version_info; diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/prims/jvmti.xml --- a/hotspot/src/share/vm/prims/jvmti.xml Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/prims/jvmti.xml Tue Feb 01 18:57:53 2011 -0800 @@ -13048,8 +13048,8 @@ - A Garbage Collection Start event is sent when a full cycle - garbage collection begins. + A Garbage Collection Start event is sent when a + garbage collection pause begins. Only stop-the-world collections are reported--that is, collections during which all threads cease to modify the state of the Java virtual machine. This means that some collectors will never generate these events. @@ -13075,8 +13075,8 @@ - A Garbage Collection Finish event is sent when a full - garbage collection cycle ends. + A Garbage Collection Finish event is sent when a + garbage collection pause ends. This event is sent while the VM is still stopped, thus the event handler must not use JNI functions and must not use functions except those which diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/prims/jvmtiEventController.cpp --- a/hotspot/src/share/vm/prims/jvmtiEventController.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/prims/jvmtiEventController.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -667,14 +667,13 @@ JvmtiEventControllerPrivate::thread_ended(JavaThread *thread) { // Removes the JvmtiThreadState associated with the specified thread. // May be called after all environments have been disposed. + assert(JvmtiThreadState_lock->is_locked(), "sanity check"); EC_TRACE(("JVMTI [%s] # thread ended", JvmtiTrace::safe_get_thread_name(thread))); JvmtiThreadState *state = thread->jvmti_thread_state(); - if (state != NULL) { - MutexLocker mu(JvmtiThreadState_lock); - delete state; - } + assert(state != NULL, "else why are we here?"); + delete state; } void JvmtiEventControllerPrivate::set_event_callbacks(JvmtiEnvBase *env, diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/prims/jvmtiExport.cpp --- a/hotspot/src/share/vm/prims/jvmtiExport.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/prims/jvmtiExport.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -2253,12 +2253,14 @@ void JvmtiExport::cleanup_thread(JavaThread* thread) { assert(JavaThread::current() == thread, "thread is not current"); - + MutexLocker mu(JvmtiThreadState_lock); - // This has to happen after the thread state is removed, which is - // why it is not in post_thread_end_event like its complement - // Maybe both these functions should be rolled into the posts? - JvmtiEventController::thread_ended(thread); + if (thread->jvmti_thread_state() != NULL) { + // This has to happen after the thread state is removed, which is + // why it is not in post_thread_end_event like its complement + // Maybe both these functions should be rolled into the posts? + JvmtiEventController::thread_ended(thread); + } } void JvmtiExport::oops_do(OopClosure* f) { @@ -2266,6 +2268,14 @@ JvmtiVMObjectAllocEventCollector::oops_do_for_all_threads(f); } +void JvmtiExport::weak_oops_do(BoolObjectClosure* is_alive, OopClosure* f) { + JvmtiTagMap::weak_oops_do(is_alive, f); +} + +void JvmtiExport::gc_epilogue() { + JvmtiCurrentBreakpoints::gc_epilogue(); +} + // Onload raw monitor transition. void JvmtiExport::transition_pending_onload_raw_monitors() { JvmtiPendingMonitors::transition_raw_monitors(); @@ -2358,15 +2368,6 @@ } #endif // SERVICES_KERNEL -// CMS has completed referencing processing so may need to update -// tag maps. -void JvmtiExport::cms_ref_processing_epilogue() { - if (JvmtiEnv::environments_might_exist()) { - JvmtiTagMap::cms_ref_processing_epilogue(); - } -} - - //////////////////////////////////////////////////////////////////////////////////////////////// // Setup current current thread for event collection. @@ -2536,36 +2537,20 @@ } }; -JvmtiGCMarker::JvmtiGCMarker(bool full) : _full(full), _invocation_count(0) { - assert(Thread::current()->is_VM_thread(), "wrong thread"); - +JvmtiGCMarker::JvmtiGCMarker() { // if there aren't any JVMTI environments then nothing to do if (!JvmtiEnv::environments_might_exist()) { return; } - if (ForceFullGCJVMTIEpilogues) { - // force 'Full GC' was done semantics for JVMTI GC epilogues - _full = true; - } - - // GarbageCollectionStart event posted from VM thread - okay because - // JVMTI is clear that the "world is stopped" and callback shouldn't - // try to call into the VM. if (JvmtiExport::should_post_garbage_collection_start()) { JvmtiExport::post_garbage_collection_start(); } - // if "full" is false it probably means this is a scavenge of the young - // generation. However it could turn out that a "full" GC is required - // so we record the number of collections so that it can be checked in - // the destructor. - if (!_full) { - _invocation_count = Universe::heap()->total_full_collections(); + if (SafepointSynchronize::is_at_safepoint()) { + // Do clean up tasks that need to be done at a safepoint + JvmtiEnvBase::check_for_periodic_clean_up(); } - - // Do clean up tasks that need to be done at a safepoint - JvmtiEnvBase::check_for_periodic_clean_up(); } JvmtiGCMarker::~JvmtiGCMarker() { @@ -2578,21 +2563,5 @@ if (JvmtiExport::should_post_garbage_collection_finish()) { JvmtiExport::post_garbage_collection_finish(); } - - // we might have initially started out doing a scavenge of the young - // generation but could have ended up doing a "full" GC - check the - // GC count to see. - if (!_full) { - _full = (_invocation_count != Universe::heap()->total_full_collections()); - } - - // Full collection probably means the perm generation has been GC'ed - // so we clear the breakpoint cache. - if (_full) { - JvmtiCurrentBreakpoints::gc_epilogue(); - } - - // Notify heap/object tagging support - JvmtiTagMap::gc_epilogue(_full); } #endif // JVMTI_KERNEL diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/prims/jvmtiExport.hpp --- a/hotspot/src/share/vm/prims/jvmtiExport.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/prims/jvmtiExport.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -346,6 +346,8 @@ static void cleanup_thread (JavaThread* thread) KERNEL_RETURN; static void oops_do(OopClosure* f) KERNEL_RETURN; + static void weak_oops_do(BoolObjectClosure* b, OopClosure* f) KERNEL_RETURN; + static void gc_epilogue() KERNEL_RETURN; static void transition_pending_onload_raw_monitors() KERNEL_RETURN; @@ -356,9 +358,6 @@ // SetNativeMethodPrefix support static char** get_all_native_method_prefixes(int* count_ptr); - - // call after CMS has completed referencing processing - static void cms_ref_processing_epilogue() KERNEL_RETURN; }; // Support class used by JvmtiDynamicCodeEventCollector and others. It @@ -492,55 +491,11 @@ // Base class for reporting GC events to JVMTI. class JvmtiGCMarker : public StackObj { - private: - bool _full; // marks a "full" GC - unsigned int _invocation_count; // GC invocation count - protected: - JvmtiGCMarker(bool full) KERNEL_RETURN; // protected - ~JvmtiGCMarker() KERNEL_RETURN; // protected + public: + JvmtiGCMarker() KERNEL_RETURN; + ~JvmtiGCMarker() KERNEL_RETURN; }; - -// Support class used to report GC events to JVMTI. The class is stack -// allocated and should be placed in the doit() implementation of all -// vm operations that do a stop-the-world GC for failed allocation. -// -// Usage :- -// -// void VM_GenCollectForAllocation::doit() { -// JvmtiGCForAllocationMarker jgcm; -// : -// } -// -// If jvmti is not enabled the constructor and destructor is essentially -// a no-op (no overhead). -// -class JvmtiGCForAllocationMarker : public JvmtiGCMarker { - public: - JvmtiGCForAllocationMarker() : JvmtiGCMarker(false) { - } -}; - -// Support class used to report GC events to JVMTI. The class is stack -// allocated and should be placed in the doit() implementation of all -// vm operations that do a "full" stop-the-world GC. This class differs -// from JvmtiGCForAllocationMarker in that this class assumes that a -// "full" GC will happen. -// -// Usage :- -// -// void VM_GenCollectFull::doit() { -// JvmtiGCFullMarker jgcm; -// : -// } -// -class JvmtiGCFullMarker : public JvmtiGCMarker { - public: - JvmtiGCFullMarker() : JvmtiGCMarker(true) { - } -}; - - // JvmtiHideSingleStepping is a helper class for hiding // internal single step events. class JvmtiHideSingleStepping : public StackObj { diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/prims/jvmtiImpl.cpp --- a/hotspot/src/share/vm/prims/jvmtiImpl.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/prims/jvmtiImpl.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -217,7 +217,6 @@ void GrowableCache::gc_epilogue() { int len = _elements->length(); - // recompute the new cache value after GC for (int i=0; iat(i)->getCacheValue(); } @@ -401,7 +400,7 @@ _bps.oops_do(f); } -void JvmtiBreakpoints::gc_epilogue() { +void JvmtiBreakpoints::gc_epilogue() { _bps.gc_epilogue(); } @@ -540,7 +539,6 @@ } } - /////////////////////////////////////////////////////////////// // // class VM_GetOrSetLocal diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/prims/jvmtiImpl.hpp --- a/hotspot/src/share/vm/prims/jvmtiImpl.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/prims/jvmtiImpl.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -117,6 +117,7 @@ void clear(); // apply f to every element and update the cache void oops_do(OopClosure* f); + // update the cache after a full gc void gc_epilogue(); }; @@ -278,13 +279,13 @@ int length(); void oops_do(OopClosure* f); - void gc_epilogue(); void print(); int set(JvmtiBreakpoint& bp); int clear(JvmtiBreakpoint& bp); void clearall_in_class_at_safepoint(klassOop klass); void clearall(); + void gc_epilogue(); }; diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/prims/jvmtiRedefineClasses.cpp --- a/hotspot/src/share/vm/prims/jvmtiRedefineClasses.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/prims/jvmtiRedefineClasses.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1458,7 +1458,7 @@ if (bc_length == 0) { // More complicated bytecodes report a length of zero so // we have to try again a slightly different way. - bc_length = Bytecodes::length_at(bcp); + bc_length = Bytecodes::length_at(method(), bcp); } assert(bc_length != 0, "impossible bytecode length"); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/prims/jvmtiTagMap.cpp --- a/hotspot/src/share/vm/prims/jvmtiTagMap.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/prims/jvmtiTagMap.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -50,7 +50,7 @@ // JvmtiTagHashmapEntry // -// Each entry encapsulates a JNI weak reference to the tagged object +// Each entry encapsulates a reference to the tagged object // and the tag value. In addition an entry includes a next pointer which // is used to chain entries together. @@ -58,24 +58,25 @@ private: friend class JvmtiTagMap; - jweak _object; // JNI weak ref to tagged object + oop _object; // tagged object jlong _tag; // the tag JvmtiTagHashmapEntry* _next; // next on the list - inline void init(jweak object, jlong tag) { + inline void init(oop object, jlong tag) { _object = object; _tag = tag; _next = NULL; } // constructor - JvmtiTagHashmapEntry(jweak object, jlong tag) { init(object, tag); } + JvmtiTagHashmapEntry(oop object, jlong tag) { init(object, tag); } public: // accessor methods - inline jweak object() const { return _object; } - inline jlong tag() const { return _tag; } + inline oop object() const { return _object; } + inline oop* object_addr() { return &_object; } + inline jlong tag() const { return _tag; } inline void set_tag(jlong tag) { assert(tag != 0, "can't be zero"); @@ -92,9 +93,7 @@ // A hashmap is essentially a table of pointers to entries. Entries // are hashed to a location, or position in the table, and then // chained from that location. The "key" for hashing is address of -// the object, or oop. The "value" is the JNI weak reference to the -// object and the tag value. Keys are not stored with the entry. -// Instead the weak reference is resolved to obtain the key. +// the object, or oop. The "value" is the tag value. // // A hashmap maintains a count of the number entries in the hashmap // and resizes if the number of entries exceeds a given threshold. @@ -206,7 +205,7 @@ JvmtiTagHashmapEntry* entry = _table[i]; while (entry != NULL) { JvmtiTagHashmapEntry* next = entry->next(); - oop key = JNIHandles::resolve(entry->object()); + oop key = entry->object(); assert(key != NULL, "jni weak reference cleared!!"); unsigned int h = hash(key, new_size); JvmtiTagHashmapEntry* anchor = new_table[h]; @@ -299,14 +298,12 @@ unsigned int h = hash(key); JvmtiTagHashmapEntry* entry = _table[h]; while (entry != NULL) { - oop orig_key = JNIHandles::resolve(entry->object()); - assert(orig_key != NULL, "jni weak reference cleared!!"); - if (key == orig_key) { - break; + if (entry->object() == key) { + return entry; } entry = entry->next(); } - return entry; + return NULL; } @@ -343,9 +340,7 @@ JvmtiTagHashmapEntry* entry = _table[h]; JvmtiTagHashmapEntry* prev = NULL; while (entry != NULL) { - oop orig_key = JNIHandles::resolve(entry->object()); - assert(orig_key != NULL, "jni weak reference cleared!!"); - if (key == orig_key) { + if (key == entry->object()) { break; } prev = entry; @@ -418,54 +413,6 @@ } } -// memory region for young generation -MemRegion JvmtiTagMap::_young_gen; - -// get the memory region used for the young generation -void JvmtiTagMap::get_young_generation() { - CollectedHeap* ch = Universe::heap(); - switch (ch->kind()) { - case (CollectedHeap::GenCollectedHeap): { - _young_gen = ((GenCollectedHeap*)ch)->get_gen(0)->reserved(); - break; - } -#ifndef SERIALGC - case (CollectedHeap::ParallelScavengeHeap): { - _young_gen = ((ParallelScavengeHeap*)ch)->young_gen()->reserved(); - break; - } - case (CollectedHeap::G1CollectedHeap): { - // Until a more satisfactory solution is implemented, all - // oops in the tag map will require rehash at each gc. - // This is a correct, if extremely inefficient solution. - // See RFE 6621729 for related commentary. - _young_gen = ch->reserved_region(); - break; - } -#endif // !SERIALGC - default: - ShouldNotReachHere(); - } -} - -// returns true if oop is in the young generation -inline bool JvmtiTagMap::is_in_young(oop o) { - assert(_young_gen.start() != NULL, "checking"); - void* p = (void*)o; - bool in_young = _young_gen.contains(p); - return in_young; -} - -// returns the appropriate hashmap for a given object -inline JvmtiTagHashmap* JvmtiTagMap::hashmap_for(oop o) { - if (is_in_young(o)) { - return _hashmap[0]; - } else { - return _hashmap[1]; - } -} - - // create a JvmtiTagMap JvmtiTagMap::JvmtiTagMap(JvmtiEnv* env) : _env(env), @@ -476,13 +423,7 @@ assert(JvmtiThreadState_lock->is_locked(), "sanity check"); assert(((JvmtiEnvBase *)env)->tag_map() == NULL, "tag map already exists for environment"); - // create the hashmaps - for (int i=0; iset_tag_map(this); @@ -496,25 +437,20 @@ // also being destroryed. ((JvmtiEnvBase *)_env)->set_tag_map(NULL); - // iterate over the hashmaps and destroy each of the entries - for (int i=0; itable(); - for (int j=0; jsize(); j++) { - JvmtiTagHashmapEntry *entry = table[j]; - while (entry != NULL) { - JvmtiTagHashmapEntry* next = entry->next(); - jweak ref = entry->object(); - JNIHandles::destroy_weak_global(ref); - delete entry; - entry = next; - } + JvmtiTagHashmapEntry** table = _hashmap->table(); + for (int j = 0; j < _hashmap->size(); j++) { + JvmtiTagHashmapEntry* entry = table[j]; + while (entry != NULL) { + JvmtiTagHashmapEntry* next = entry->next(); + delete entry; + entry = next; } - - // finally destroy the hashmap - delete hashmap; } + // finally destroy the hashmap + delete _hashmap; + _hashmap = NULL; + // remove any entries on the free list JvmtiTagHashmapEntry* entry = _free_entries; while (entry != NULL) { @@ -522,12 +458,13 @@ delete entry; entry = next; } + _free_entries = NULL; } // create a hashmap entry // - if there's an entry on the (per-environment) free list then this // is returned. Otherwise an new entry is allocated. -JvmtiTagHashmapEntry* JvmtiTagMap::create_entry(jweak ref, jlong tag) { +JvmtiTagHashmapEntry* JvmtiTagMap::create_entry(oop ref, jlong tag) { assert(Thread::current()->is_VM_thread() || is_locked(), "checking"); JvmtiTagHashmapEntry* entry; if (_free_entries == NULL) { @@ -558,10 +495,10 @@ // returns the tag map for the given environments. If the tag map // doesn't exist then it is created. JvmtiTagMap* JvmtiTagMap::tag_map_for(JvmtiEnv* env) { - JvmtiTagMap* tag_map = ((JvmtiEnvBase *)env)->tag_map(); + JvmtiTagMap* tag_map = ((JvmtiEnvBase*)env)->tag_map(); if (tag_map == NULL) { MutexLocker mu(JvmtiThreadState_lock); - tag_map = ((JvmtiEnvBase *)env)->tag_map(); + tag_map = ((JvmtiEnvBase*)env)->tag_map(); if (tag_map == NULL) { tag_map = new JvmtiTagMap(env); } @@ -573,17 +510,13 @@ // iterate over all entries in the tag map. void JvmtiTagMap::entry_iterate(JvmtiTagHashmapEntryClosure* closure) { - for (int i=0; ientry_iterate(closure); - } + hashmap()->entry_iterate(closure); } // returns true if the hashmaps are empty bool JvmtiTagMap::is_empty() { assert(SafepointSynchronize::is_at_safepoint() || is_locked(), "checking"); - assert(n_hashmaps == 2, "not implemented"); - return ((_hashmap[0]->entry_count() == 0) && (_hashmap[1]->entry_count() == 0)); + return hashmap()->entry_count() == 0; } @@ -591,7 +524,7 @@ // not tagged // static inline jlong tag_for(JvmtiTagMap* tag_map, oop o) { - JvmtiTagHashmapEntry* entry = tag_map->hashmap_for(o)->find(o); + JvmtiTagHashmapEntry* entry = tag_map->hashmap()->find(o); if (entry == NULL) { return 0; } else { @@ -655,7 +588,7 @@ // record the context _tag_map = tag_map; - _hashmap = tag_map->hashmap_for(_o); + _hashmap = tag_map->hashmap(); _entry = _hashmap->find(_o); // get object tag @@ -694,23 +627,18 @@ if (obj_tag != 0) { // callback has tagged the object assert(Thread::current()->is_VM_thread(), "must be VMThread"); - HandleMark hm; - Handle h(o); - jweak ref = JNIHandles::make_weak_global(h); - entry = tag_map()->create_entry(ref, obj_tag); + entry = tag_map()->create_entry(o, obj_tag); hashmap->add(o, entry); } } else { // object was previously tagged - the callback may have untagged // the object or changed the tag value if (obj_tag == 0) { - jweak ref = entry->object(); JvmtiTagHashmapEntry* entry_removed = hashmap->remove(o); assert(entry_removed == entry, "checking"); tag_map()->destroy_entry(entry); - JNIHandles::destroy_weak_global(ref); } else { if (obj_tag != entry->tag()) { entry->set_tag(obj_tag); @@ -760,7 +688,7 @@ // for Classes the klassOop is tagged _referrer = klassOop_if_java_lang_Class(referrer); // record the context - _referrer_hashmap = tag_map->hashmap_for(_referrer); + _referrer_hashmap = tag_map->hashmap(); _referrer_entry = _referrer_hashmap->find(_referrer); // get object tag @@ -796,8 +724,7 @@ // // This function is performance critical. If many threads attempt to tag objects // around the same time then it's possible that the Mutex associated with the -// tag map will be a hot lock. Eliminating this lock will not eliminate the issue -// because creating a JNI weak reference requires acquiring a global lock also. +// tag map will be a hot lock. void JvmtiTagMap::set_tag(jobject object, jlong tag) { MutexLocker ml(lock()); @@ -808,22 +735,14 @@ o = klassOop_if_java_lang_Class(o); // see if the object is already tagged - JvmtiTagHashmap* hashmap = hashmap_for(o); + JvmtiTagHashmap* hashmap = _hashmap; JvmtiTagHashmapEntry* entry = hashmap->find(o); // if the object is not already tagged then we tag it if (entry == NULL) { if (tag != 0) { - HandleMark hm; - Handle h(o); - jweak ref = JNIHandles::make_weak_global(h); - - // the object may have moved because make_weak_global may - // have blocked - thus it is necessary resolve the handle - // and re-hash the object. - o = h(); - entry = create_entry(ref, tag); - hashmap_for(o)->add(o, entry); + entry = create_entry(o, tag); + hashmap->add(o, entry); } else { // no-op } @@ -831,13 +750,9 @@ // if the object is already tagged then we either update // the tag (if a new tag value has been provided) // or remove the object if the new tag value is 0. - // Removing the object requires that we also delete the JNI - // weak ref to the object. if (tag == 0) { - jweak ref = entry->object(); hashmap->remove(o); destroy_entry(entry); - JNIHandles::destroy_weak_global(ref); } else { entry->set_tag(tag); } @@ -1626,8 +1541,8 @@ void do_entry(JvmtiTagHashmapEntry* entry) { for (int i=0; i<_tag_count; i++) { if (_tags[i] == entry->tag()) { - oop o = JNIHandles::resolve(entry->object()); - assert(o != NULL && o != JNIHandles::deleted_handle(), "sanity check"); + oop o = entry->object(); + assert(o != NULL, "sanity check"); // the mirror is tagged if (o->is_klass()) { @@ -3374,62 +3289,25 @@ } -// called post-GC -// - for each JVMTI environment with an object tag map, call its rehash -// function to re-sync with the new object locations. -void JvmtiTagMap::gc_epilogue(bool full) { - assert(SafepointSynchronize::is_at_safepoint(), "must be executed at a safepoint"); +void JvmtiTagMap::weak_oops_do(BoolObjectClosure* is_alive, OopClosure* f) { + // No locks during VM bring-up (0 threads) and no safepoints after main + // thread creation and before VMThread creation (1 thread); initial GC + // verification can happen in that window which gets to here. + assert(Threads::number_of_threads() <= 1 || + SafepointSynchronize::is_at_safepoint(), + "must be executed at a safepoint"); if (JvmtiEnv::environments_might_exist()) { - // re-obtain the memory region for the young generation (might - // changed due to adaptive resizing policy) - get_young_generation(); - JvmtiEnvIterator it; for (JvmtiEnvBase* env = it.first(); env != NULL; env = it.next(env)) { JvmtiTagMap* tag_map = env->tag_map(); if (tag_map != NULL && !tag_map->is_empty()) { - TraceTime t(full ? "JVMTI Full Rehash " : "JVMTI Rehash ", TraceJVMTIObjectTagging); - if (full) { - tag_map->rehash(0, n_hashmaps); - } else { - tag_map->rehash(0, 0); // tag map for young gen only - } + tag_map->do_weak_oops(is_alive, f); } } } } -// CMS has completed referencing processing so we may have JNI weak refs -// to objects in the CMS generation that have been GC'ed. -void JvmtiTagMap::cms_ref_processing_epilogue() { - assert(SafepointSynchronize::is_at_safepoint(), "must be executed at a safepoint"); - assert(UseConcMarkSweepGC, "should only be used with CMS"); - if (JvmtiEnv::environments_might_exist()) { - JvmtiEnvIterator it; - for (JvmtiEnvBase* env = it.first(); env != NULL; env = it.next(env)) { - JvmtiTagMap* tag_map = ((JvmtiEnvBase *)env)->tag_map(); - if (tag_map != NULL && !tag_map->is_empty()) { - TraceTime t("JVMTI Rehash (CMS) ", TraceJVMTIObjectTagging); - tag_map->rehash(1, n_hashmaps); // assume CMS not used in young gen - } - } - } -} - - -// For each entry in the hashmaps 'start' to 'end' : -// -// 1. resolve the JNI weak reference -// -// 2. If it resolves to NULL it means the object has been freed so the entry -// is removed, the weak reference destroyed, and the object free event is -// posted (if enabled). -// -// 3. If the weak reference resolves to an object then we re-hash the object -// to see if it has moved or has been promoted (from the young to the old -// generation for example). -// -void JvmtiTagMap::rehash(int start, int end) { +void JvmtiTagMap::do_weak_oops(BoolObjectClosure* is_alive, OopClosure* f) { // does this environment have the OBJECT_FREE event enabled bool post_object_free = env()->is_enabled(JVMTI_EVENT_OBJECT_FREE); @@ -3437,143 +3315,98 @@ // counters used for trace message int freed = 0; int moved = 0; - int promoted = 0; - - // we assume there are two hashmaps - one for the young generation - // and the other for all other spaces. - assert(n_hashmaps == 2, "not implemented"); - JvmtiTagHashmap* young_hashmap = _hashmap[0]; - JvmtiTagHashmap* other_hashmap = _hashmap[1]; + + JvmtiTagHashmap* hashmap = this->hashmap(); // reenable sizing (if disabled) - young_hashmap->set_resizing_enabled(true); - other_hashmap->set_resizing_enabled(true); - - // when re-hashing the hashmap corresponding to the young generation we - // collect the entries corresponding to objects that have been promoted. - JvmtiTagHashmapEntry* promoted_entries = NULL; - - if (end >= n_hashmaps) { - end = n_hashmaps - 1; + hashmap->set_resizing_enabled(true); + + // if the hashmap is empty then we can skip it + if (hashmap->_entry_count == 0) { + return; } - for (int i=start; i <= end; i++) { - JvmtiTagHashmap* hashmap = _hashmap[i]; - - // if the hashmap is empty then we can skip it - if (hashmap->_entry_count == 0) { - continue; - } - - // now iterate through each entry in the table - - JvmtiTagHashmapEntry** table = hashmap->table(); - int size = hashmap->size(); - - for (int pos=0; posnext(); - - jweak ref = entry->object(); - oop oop = JNIHandles::resolve(ref); - - // has object been GC'ed - if (oop == NULL) { - // grab the tag - jlong tag = entry->tag(); - guarantee(tag != 0, "checking"); - - // remove GC'ed entry from hashmap and return the - // entry to the free list - hashmap->remove(prev, pos, entry); - destroy_entry(entry); - - // destroy the weak ref - JNIHandles::destroy_weak_global(ref); - - // post the event to the profiler - if (post_object_free) { - JvmtiExport::post_object_free(env(), tag); + // now iterate through each entry in the table + + JvmtiTagHashmapEntry** table = hashmap->table(); + int size = hashmap->size(); + + JvmtiTagHashmapEntry* delayed_add = NULL; + + for (int pos = 0; pos < size; ++pos) { + JvmtiTagHashmapEntry* entry = table[pos]; + JvmtiTagHashmapEntry* prev = NULL; + + while (entry != NULL) { + JvmtiTagHashmapEntry* next = entry->next(); + + oop* obj = entry->object_addr(); + + // has object been GC'ed + if (!is_alive->do_object_b(entry->object())) { + // grab the tag + jlong tag = entry->tag(); + guarantee(tag != 0, "checking"); + + // remove GC'ed entry from hashmap and return the + // entry to the free list + hashmap->remove(prev, pos, entry); + destroy_entry(entry); + + // post the event to the profiler + if (post_object_free) { + JvmtiExport::post_object_free(env(), tag); + } + + ++freed; + } else { + f->do_oop(entry->object_addr()); + oop new_oop = entry->object(); + + // if the object has moved then re-hash it and move its + // entry to its new location. + unsigned int new_pos = JvmtiTagHashmap::hash(new_oop, size); + if (new_pos != (unsigned int)pos) { + if (prev == NULL) { + table[pos] = next; + } else { + prev->set_next(next); } - - freed++; - entry = next; - continue; - } - - // if this is the young hashmap then the object is either promoted - // or moved. - // if this is the other hashmap then the object is moved. - - bool same_gen; - if (i == 0) { - assert(hashmap == young_hashmap, "checking"); - same_gen = is_in_young(oop); - } else { - same_gen = true; - } - - - if (same_gen) { - // if the object has moved then re-hash it and move its - // entry to its new location. - unsigned int new_pos = JvmtiTagHashmap::hash(oop, size); - if (new_pos != (unsigned int)pos) { - if (prev == NULL) { - table[pos] = next; - } else { - prev->set_next(next); - } + if (new_pos < (unsigned int)pos) { entry->set_next(table[new_pos]); table[new_pos] = entry; - moved++; } else { - // object didn't move - prev = entry; + // Delay adding this entry to it's new position as we'd end up + // hitting it again during this iteration. + entry->set_next(delayed_add); + delayed_add = entry; } + moved++; } else { - // object has been promoted so remove the entry from the - // young hashmap - assert(hashmap == young_hashmap, "checking"); - hashmap->remove(prev, pos, entry); - - // move the entry to the promoted list - entry->set_next(promoted_entries); - promoted_entries = entry; + // object didn't move + prev = entry; } - - entry = next; } + + entry = next; } } - - // add the entries, corresponding to the promoted objects, to the - // other hashmap. - JvmtiTagHashmapEntry* entry = promoted_entries; - while (entry != NULL) { - oop o = JNIHandles::resolve(entry->object()); - assert(hashmap_for(o) == other_hashmap, "checking"); - JvmtiTagHashmapEntry* next = entry->next(); - other_hashmap->add(o, entry); - entry = next; - promoted++; + // Re-add all the entries which were kept aside + while (delayed_add != NULL) { + JvmtiTagHashmapEntry* next = delayed_add->next(); + unsigned int pos = JvmtiTagHashmap::hash(delayed_add->object(), size); + delayed_add->set_next(table[pos]); + table[pos] = delayed_add; + delayed_add = next; } // stats if (TraceJVMTIObjectTagging) { - int total_moves = promoted + moved; - - int post_total = 0; - for (int i=0; i_entry_count; - } + int post_total = hashmap->_entry_count; int pre_total = post_total + freed; - tty->print("(%d->%d, %d freed, %d promoted, %d total moves)", - pre_total, post_total, freed, promoted, total_moves); + tty->print_cr("(%d->%d, %d freed, %d total moves)", + pre_total, post_total, freed, moved); } } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/prims/jvmtiTagMap.hpp --- a/hotspot/src/share/vm/prims/jvmtiTagMap.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/prims/jvmtiTagMap.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -45,17 +45,12 @@ private: enum{ - n_hashmaps = 2, // encapsulates 2 hashmaps - max_free_entries = 4096 // maximum number of free entries per env + max_free_entries = 4096 // maximum number of free entries per env }; - // memory region for young generation - static MemRegion _young_gen; - static void get_young_generation(); - JvmtiEnv* _env; // the jvmti environment Mutex _lock; // lock for this tag map - JvmtiTagHashmap* _hashmap[n_hashmaps]; // the hashmaps + JvmtiTagHashmap* _hashmap; // the hashmap JvmtiTagHashmapEntry* _free_entries; // free list for this environment int _free_entries_count; // number of entries on the free list @@ -67,11 +62,7 @@ inline Mutex* lock() { return &_lock; } inline JvmtiEnv* env() const { return _env; } - // rehash tags maps for generation start to end - void rehash(int start, int end); - - // indicates if the object is in the young generation - static bool is_in_young(oop o); + void do_weak_oops(BoolObjectClosure* is_alive, OopClosure* f); // iterate over all entries in this tag map void entry_iterate(JvmtiTagHashmapEntryClosure* closure); @@ -81,11 +72,10 @@ // indicates if this tag map is locked bool is_locked() { return lock()->is_locked(); } - // return the appropriate hashmap for a given object - JvmtiTagHashmap* hashmap_for(oop o); + JvmtiTagHashmap* hashmap() { return _hashmap; } // create/destroy entries - JvmtiTagHashmapEntry* create_entry(jweak ref, jlong tag); + JvmtiTagHashmapEntry* create_entry(oop ref, jlong tag); void destroy_entry(JvmtiTagHashmapEntry* entry); // returns true if the hashmaps are empty @@ -134,11 +124,8 @@ jint* count_ptr, jobject** object_result_ptr, jlong** tag_result_ptr); - // call post-GC to rehash the tag maps. - static void gc_epilogue(bool full); - - // call after referencing processing has completed (CMS) - static void cms_ref_processing_epilogue(); + static void weak_oops_do( + BoolObjectClosure* is_alive, OopClosure* f) KERNEL_RETURN; }; #endif // SHARE_VM_PRIMS_JVMTITAGMAP_HPP diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/prims/methodComparator.cpp --- a/hotspot/src/share/vm/prims/methodComparator.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/prims/methodComparator.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -194,10 +194,10 @@ case Bytecodes::_ldc : // fall through case Bytecodes::_ldc_w : { - Bytecode_loadconstant* ldc_old = Bytecode_loadconstant_at(_s_old->method(), _s_old->bci()); - Bytecode_loadconstant* ldc_new = Bytecode_loadconstant_at(_s_new->method(), _s_new->bci()); - int cpi_old = ldc_old->pool_index(); - int cpi_new = ldc_new->pool_index(); + Bytecode_loadconstant ldc_old(_s_old->method(), _s_old->bci()); + Bytecode_loadconstant ldc_new(_s_new->method(), _s_new->bci()); + int cpi_old = ldc_old.pool_index(); + int cpi_new = ldc_new.pool_index(); if (!pool_constants_same(cpi_old, cpi_new)) return false; break; @@ -267,8 +267,8 @@ case Bytecodes::_ifnonnull : // fall through case Bytecodes::_ifnull : // fall through case Bytecodes::_jsr : { - int old_ofs = _s_old->bytecode()->get_offset_s2(c_old); - int new_ofs = _s_new->bytecode()->get_offset_s2(c_new); + int old_ofs = _s_old->bytecode().get_offset_s2(c_old); + int new_ofs = _s_new->bytecode().get_offset_s2(c_new); if (_switchable_test) { int old_dest = _s_old->bci() + old_ofs; int new_dest = _s_new->bci() + new_ofs; @@ -304,8 +304,8 @@ case Bytecodes::_goto_w : // fall through case Bytecodes::_jsr_w : { - int old_ofs = _s_old->bytecode()->get_offset_s4(c_old); - int new_ofs = _s_new->bytecode()->get_offset_s4(c_new); + int old_ofs = _s_old->bytecode().get_offset_s4(c_old); + int new_ofs = _s_new->bytecode().get_offset_s4(c_new); if (_switchable_test) { int old_dest = _s_old->bci() + old_ofs; int new_dest = _s_new->bci() + new_ofs; diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/prims/methodHandleWalk.cpp --- a/hotspot/src/share/vm/prims/methodHandleWalk.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/prims/methodHandleWalk.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -137,7 +137,6 @@ void MethodHandleChain::lose(const char* msg, TRAPS) { - assert(false, "lose"); _lose_message = msg; if (!THREAD->is_Java_thread() || ((JavaThread*)THREAD)->thread_state() != _thread_in_vm) { // throw a preallocated exception diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/prims/methodHandles.cpp --- a/hotspot/src/share/vm/prims/methodHandles.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/prims/methodHandles.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -111,7 +111,7 @@ //------------------------------------------------------------------------------ // MethodHandles::generate_adapters // -void MethodHandles::generate_adapters(TRAPS) { +void MethodHandles::generate_adapters() { if (!EnableMethodHandles || SystemDictionary::MethodHandle_klass() == NULL) return; assert(_adapter_code == NULL, "generate only once"); @@ -123,20 +123,20 @@ vm_exit_out_of_memory(_adapter_code_size, "CodeCache: no room for MethodHandles adapters"); CodeBuffer code(_adapter_code); MethodHandlesAdapterGenerator g(&code); - g.generate(CHECK); + g.generate(); } //------------------------------------------------------------------------------ // MethodHandlesAdapterGenerator::generate // -void MethodHandlesAdapterGenerator::generate(TRAPS) { +void MethodHandlesAdapterGenerator::generate() { // Generate generic method handle adapters. for (MethodHandles::EntryKind ek = MethodHandles::_EK_FIRST; ek < MethodHandles::_EK_LIMIT; ek = MethodHandles::EntryKind(1 + (int)ek)) { StubCodeMark mark(this, "MethodHandle", MethodHandles::entry_name(ek)); - MethodHandles::generate_method_handle_stub(_masm, ek, CHECK); + MethodHandles::generate_method_handle_stub(_masm, ek); } } @@ -2621,10 +2621,20 @@ warning("JSR 292 method handle code is mismatched to this JVM. Disabling support."); enable_MH = false; } + } else { + enable_MH = false; } } if (enable_MH) { + // We need to link the MethodHandleImpl klass before we generate + // the method handle adapters as the _raise_exception adapter uses + // one of its methods (and its c2i-adapter). + KlassHandle k = SystemDictionaryHandles::MethodHandleImpl_klass(); + instanceKlass* ik = instanceKlass::cast(k()); + ik->link_class(CHECK); + + MethodHandles::generate_adapters(); MethodHandles::set_enabled(true); } @@ -2645,10 +2655,5 @@ MethodHandles::set_enabled(true); } } - - // Generate method handles adapters if enabled. - if (MethodHandles::enabled()) { - MethodHandles::generate_adapters(CHECK); - } } JVM_END diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/prims/methodHandles.hpp --- a/hotspot/src/share/vm/prims/methodHandles.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/prims/methodHandles.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -294,11 +294,11 @@ enum { _suppress_defc = 1, _suppress_name = 2, _suppress_type = 4 }; // Generate MethodHandles adapters. - static void generate_adapters(TRAPS); + static void generate_adapters(); // Called from InterpreterGenerator and MethodHandlesAdapterGenerator. static address generate_method_handle_interpreter_entry(MacroAssembler* _masm); - static void generate_method_handle_stub(MacroAssembler* _masm, EntryKind ek, TRAPS); + static void generate_method_handle_stub(MacroAssembler* _masm, EntryKind ek); // argument list parsing static int argument_slot(oop method_type, int arg); @@ -530,7 +530,7 @@ public: MethodHandlesAdapterGenerator(CodeBuffer* code) : StubCodeGenerator(code) {} - void generate(TRAPS); + void generate(); }; #endif // SHARE_VM_PRIMS_METHODHANDLES_HPP diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/prims/unsafe.cpp --- a/hotspot/src/share/vm/prims/unsafe.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/prims/unsafe.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -154,12 +154,11 @@ #define GET_FIELD_VOLATILE(obj, offset, type_name, v) \ oop p = JNIHandles::resolve(obj); \ - volatile type_name v = *(volatile type_name*)index_oop_from_field_offset_long(p, offset) + volatile type_name v = OrderAccess::load_acquire((volatile type_name*)index_oop_from_field_offset_long(p, offset)); #define SET_FIELD_VOLATILE(obj, offset, type_name, x) \ oop p = JNIHandles::resolve(obj); \ - *(volatile type_name*)index_oop_from_field_offset_long(p, offset) = x; \ - OrderAccess::fence(); + OrderAccess::release_store_fence((volatile type_name*)index_oop_from_field_offset_long(p, offset), x); // Macros for oops that check UseCompressedOops @@ -181,7 +180,8 @@ v = oopDesc::decode_heap_oop(n); \ } else { \ v = *(volatile oop*)index_oop_from_field_offset_long(p, offset); \ - } + } \ + OrderAccess::acquire(); // Get/SetObject must be special-cased, since it works with handles. @@ -248,14 +248,22 @@ UnsafeWrapper("Unsafe_SetObjectVolatile"); oop x = JNIHandles::resolve(x_h); oop p = JNIHandles::resolve(obj); + void* addr = index_oop_from_field_offset_long(p, offset); + OrderAccess::release(); if (UseCompressedOops) { - oop_store((narrowOop*)index_oop_from_field_offset_long(p, offset), x); + oop_store((narrowOop*)addr, x); } else { - oop_store((oop*)index_oop_from_field_offset_long(p, offset), x); + oop_store((oop*)addr, x); } OrderAccess::fence(); UNSAFE_END +#if defined(SPARC) || defined(X86) +// Sparc and X86 have atomic jlong (8 bytes) instructions + +#else +// Keep old code for platforms which may not have atomic jlong (8 bytes) instructions + // Volatile long versions must use locks if !VM_Version::supports_cx8(). // support_cx8 is a surrogate for 'supports atomic long memory ops'. @@ -291,6 +299,7 @@ } UNSAFE_END +#endif // not SPARC and not X86 #define DEFINE_GETSETOOP(jboolean, Boolean) \ \ @@ -320,6 +329,16 @@ \ // END DEFINE_GETSETOOP. +DEFINE_GETSETOOP(jboolean, Boolean) +DEFINE_GETSETOOP(jbyte, Byte) +DEFINE_GETSETOOP(jshort, Short); +DEFINE_GETSETOOP(jchar, Char); +DEFINE_GETSETOOP(jint, Int); +DEFINE_GETSETOOP(jlong, Long); +DEFINE_GETSETOOP(jfloat, Float); +DEFINE_GETSETOOP(jdouble, Double); + +#undef DEFINE_GETSETOOP #define DEFINE_GETSETOOP_VOLATILE(jboolean, Boolean) \ \ @@ -336,47 +355,49 @@ \ // END DEFINE_GETSETOOP_VOLATILE. -DEFINE_GETSETOOP(jboolean, Boolean) -DEFINE_GETSETOOP(jbyte, Byte) -DEFINE_GETSETOOP(jshort, Short); -DEFINE_GETSETOOP(jchar, Char); -DEFINE_GETSETOOP(jint, Int); -DEFINE_GETSETOOP(jlong, Long); -DEFINE_GETSETOOP(jfloat, Float); -DEFINE_GETSETOOP(jdouble, Double); - DEFINE_GETSETOOP_VOLATILE(jboolean, Boolean) DEFINE_GETSETOOP_VOLATILE(jbyte, Byte) DEFINE_GETSETOOP_VOLATILE(jshort, Short); DEFINE_GETSETOOP_VOLATILE(jchar, Char); DEFINE_GETSETOOP_VOLATILE(jint, Int); -// no long -- handled specially DEFINE_GETSETOOP_VOLATILE(jfloat, Float); DEFINE_GETSETOOP_VOLATILE(jdouble, Double); -#undef DEFINE_GETSETOOP +#if defined(SPARC) || defined(X86) +// Sparc and X86 have atomic jlong (8 bytes) instructions +DEFINE_GETSETOOP_VOLATILE(jlong, Long); +#endif + +#undef DEFINE_GETSETOOP_VOLATILE // The non-intrinsified versions of setOrdered just use setVolatile -UNSAFE_ENTRY(void, Unsafe_SetOrderedInt(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jint x)) \ - UnsafeWrapper("Unsafe_SetOrderedInt"); \ - SET_FIELD_VOLATILE(obj, offset, jint, x); \ +UNSAFE_ENTRY(void, Unsafe_SetOrderedInt(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jint x)) + UnsafeWrapper("Unsafe_SetOrderedInt"); + SET_FIELD_VOLATILE(obj, offset, jint, x); UNSAFE_END UNSAFE_ENTRY(void, Unsafe_SetOrderedObject(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject x_h)) UnsafeWrapper("Unsafe_SetOrderedObject"); oop x = JNIHandles::resolve(x_h); oop p = JNIHandles::resolve(obj); + void* addr = index_oop_from_field_offset_long(p, offset); + OrderAccess::release(); if (UseCompressedOops) { - oop_store((narrowOop*)index_oop_from_field_offset_long(p, offset), x); + oop_store((narrowOop*)addr, x); } else { - oop_store((oop*)index_oop_from_field_offset_long(p, offset), x); + oop_store((oop*)addr, x); } OrderAccess::fence(); UNSAFE_END UNSAFE_ENTRY(void, Unsafe_SetOrderedLong(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jlong x)) UnsafeWrapper("Unsafe_SetOrderedLong"); +#if defined(SPARC) || defined(X86) + // Sparc and X86 have atomic jlong (8 bytes) instructions + SET_FIELD_VOLATILE(obj, offset, jlong, x); +#else + // Keep old code for platforms which may not have atomic long (8 bytes) instructions { if (VM_Version::supports_cx8()) { SET_FIELD_VOLATILE(obj, offset, jlong, x); @@ -388,6 +409,7 @@ *addr = x; } } +#endif UNSAFE_END ////// Data in the C heap. diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/runtime/arguments.cpp --- a/hotspot/src/share/vm/runtime/arguments.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/runtime/arguments.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -918,9 +918,7 @@ } else if (strcmp(key, "sun.java.command") == 0) { _java_command = value; - // don't add this property to the properties exposed to the java application - FreeHeap(key); - return true; + // Record value in Arguments, but let it get passed to Java. } else if (strcmp(key, "sun.java.launcher.pid") == 0) { // launcher.pid property is private and is processed // in process_sun_java_launcher_properties(); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/runtime/deoptimization.cpp --- a/hotspot/src/share/vm/runtime/deoptimization.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/runtime/deoptimization.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -395,8 +395,8 @@ { HandleMark hm; methodHandle method(thread, array->element(0)->method()); - Bytecode_invoke* invoke = Bytecode_invoke_at_check(method, array->element(0)->bci()); - return_type = (invoke != NULL) ? invoke->result_type(thread) : T_ILLEGAL; + Bytecode_invoke invoke = Bytecode_invoke_check(method, array->element(0)->bci()); + return_type = invoke.is_valid() ? invoke.result_type(thread) : T_ILLEGAL; } // Compute information for handling adapters and adjusting the frame size of the caller. @@ -600,8 +600,8 @@ cur_code == Bytecodes::_invokespecial || cur_code == Bytecodes::_invokestatic || cur_code == Bytecodes::_invokeinterface) { - Bytecode_invoke* invoke = Bytecode_invoke_at(mh, iframe->interpreter_frame_bci()); - symbolHandle signature(thread, invoke->signature()); + Bytecode_invoke invoke(mh, iframe->interpreter_frame_bci()); + symbolHandle signature(thread, invoke.signature()); ArgumentSizeComputer asc(signature); cur_invoke_parameter_size = asc.size(); if (cur_code != Bytecodes::_invokestatic) { @@ -963,7 +963,7 @@ if (bci == SynchronizationEntryBCI) { code_name = "sync entry"; } else { - Bytecodes::Code code = Bytecodes::code_at(vf->method(), bci); + Bytecodes::Code code = vf->method()->code_at(bci); code_name = Bytecodes::name(code); } tty->print(" - %s", code_name); @@ -1224,7 +1224,7 @@ ScopeDesc* trap_scope = cvf->scope(); methodHandle trap_method = trap_scope->method(); int trap_bci = trap_scope->bci(); - Bytecodes::Code trap_bc = Bytecode_at(trap_method->bcp_from(trap_bci))->java_code(); + Bytecodes::Code trap_bc = trap_method->java_code_at(trap_bci); // Record this event in the histogram. gather_statistics(reason, action, trap_bc); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/runtime/frame.cpp --- a/hotspot/src/share/vm/runtime/frame.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/runtime/frame.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -930,10 +930,10 @@ // This is used sometimes for calling into the VM, not for another // interpreted or compiled frame. if (!m->is_native()) { - Bytecode_invoke *call = Bytecode_invoke_at_check(m, bci); - if (call != NULL) { - signature = symbolHandle(thread, call->signature()); - has_receiver = call->has_receiver(); + Bytecode_invoke call = Bytecode_invoke_check(m, bci); + if (call.is_valid()) { + signature = symbolHandle(thread, call.signature()); + has_receiver = call.has_receiver(); if (map->include_argument_oops() && interpreter_frame_expression_stack_size() > 0) { ResourceMark rm(thread); // is this right ??? diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/runtime/globals.hpp --- a/hotspot/src/share/vm/runtime/globals.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/runtime/globals.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1198,9 +1198,6 @@ product(ccstr, TraceJVMTI, NULL, \ "Trace flags for JVMTI functions and events") \ \ - product(bool, ForceFullGCJVMTIEpilogues, false, \ - "Force 'Full GC' was done semantics for JVMTI GC epilogues") \ - \ /* This option can change an EMCP method into an obsolete method. */ \ /* This can affect tests that except specific methods to be EMCP. */ \ /* This option should be used with caution. */ \ diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/runtime/java.cpp --- a/hotspot/src/share/vm/runtime/java.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/runtime/java.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -663,7 +663,8 @@ } _current = JDK_Version(major, minor, micro, info.update_version, info.special_update_version, build, - info.thread_park_blocker == 1); + info.thread_park_blocker == 1, + info.post_vm_init_hook_enabled == 1); } } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/runtime/java.hpp --- a/hotspot/src/share/vm/runtime/java.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/runtime/java.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -92,6 +92,7 @@ bool _partially_initialized; bool _thread_park_blocker; + bool _post_vm_init_hook_enabled; bool is_valid() const { return (_major != 0 || _partially_initialized); @@ -113,14 +114,15 @@ JDK_Version() : _major(0), _minor(0), _micro(0), _update(0), _special(0), _build(0), _partially_initialized(false), - _thread_park_blocker(false) {} + _thread_park_blocker(false), _post_vm_init_hook_enabled(false) {} JDK_Version(uint8_t major, uint8_t minor = 0, uint8_t micro = 0, uint8_t update = 0, uint8_t special = 0, uint8_t build = 0, - bool thread_park_blocker = false) : + bool thread_park_blocker = false, bool post_vm_init_hook_enabled = false) : _major(major), _minor(minor), _micro(micro), _update(update), _special(special), _build(build), _partially_initialized(false), - _thread_park_blocker(thread_park_blocker) {} + _thread_park_blocker(thread_park_blocker), + _post_vm_init_hook_enabled(post_vm_init_hook_enabled) {} // Returns the current running JDK version static JDK_Version current() { return _current; } @@ -144,6 +146,9 @@ bool supports_thread_park_blocker() const { return _thread_park_blocker; } + bool post_vm_init_hook_enabled() const { + return _post_vm_init_hook_enabled; + } // Performs a full ordering comparison using all fields (update, build, etc.) int compare(const JDK_Version& other) const; diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/runtime/jniHandles.cpp --- a/hotspot/src/share/vm/runtime/jniHandles.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/runtime/jniHandles.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "classfile/systemDictionary.hpp" #include "oops/oop.inline.hpp" +#include "prims/jvmtiExport.hpp" #include "runtime/jniHandles.hpp" #include "runtime/mutexLocker.hpp" #ifdef TARGET_OS_FAMILY_linux @@ -428,6 +429,12 @@ break; } } + + /* + * JVMTI data structures may also contain weak oops. The iteration of them + * is placed here so that we don't need to add it to each of the collectors. + */ + JvmtiExport::weak_oops_do(is_alive, f); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/runtime/mutexLocker.cpp --- a/hotspot/src/share/vm/runtime/mutexLocker.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/runtime/mutexLocker.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -80,8 +80,6 @@ Monitor* iCMS_lock = NULL; Monitor* FullGCCount_lock = NULL; Monitor* CMark_lock = NULL; -Monitor* ZF_mon = NULL; -Monitor* Cleanup_mon = NULL; Mutex* CMRegionStack_lock = NULL; Mutex* SATB_Q_FL_lock = NULL; Monitor* SATB_Q_CBL_mon = NULL; @@ -122,6 +120,9 @@ Mutex* PerfDataManager_lock = NULL; Mutex* OopMapCacheAlloc_lock = NULL; +Mutex* FreeList_lock = NULL; +Monitor* SecondaryFreeList_lock = NULL; +Mutex* OldSets_lock = NULL; Mutex* MMUTracker_lock = NULL; Mutex* HotCardCache_lock = NULL; @@ -177,8 +178,6 @@ } if (UseG1GC) { def(CMark_lock , Monitor, nonleaf, true ); // coordinate concurrent mark thread - def(ZF_mon , Monitor, leaf, true ); - def(Cleanup_mon , Monitor, nonleaf, true ); def(CMRegionStack_lock , Mutex, leaf, true ); def(SATB_Q_FL_lock , Mutex , special, true ); def(SATB_Q_CBL_mon , Monitor, nonleaf, true ); @@ -188,6 +187,9 @@ def(DirtyCardQ_CBL_mon , Monitor, nonleaf, true ); def(Shared_DirtyCardQ_lock , Mutex, nonleaf, true ); + def(FreeList_lock , Mutex, leaf , true ); + def(SecondaryFreeList_lock , Monitor, leaf , true ); + def(OldSets_lock , Mutex , leaf , true ); def(MMUTracker_lock , Mutex , leaf , true ); def(HotCardCache_lock , Mutex , special , true ); def(EvacFailureStack_lock , Mutex , nonleaf , true ); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/runtime/mutexLocker.hpp --- a/hotspot/src/share/vm/runtime/mutexLocker.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/runtime/mutexLocker.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -76,8 +76,6 @@ extern Monitor* iCMS_lock; // CMS incremental mode start/stop notification extern Monitor* FullGCCount_lock; // in support of "concurrent" full gc extern Monitor* CMark_lock; // used for concurrent mark thread coordination -extern Monitor* ZF_mon; // used for G1 conc zero-fill. -extern Monitor* Cleanup_mon; // used for G1 conc cleanup. extern Mutex* CMRegionStack_lock; // used for protecting accesses to the CM region stack extern Mutex* SATB_Q_FL_lock; // Protects SATB Q // buffer free list. @@ -125,6 +123,9 @@ extern Mutex* ParkerFreeList_lock; extern Mutex* OopMapCacheAlloc_lock; // protects allocation of oop_map caches +extern Mutex* FreeList_lock; // protects the free region list during safepoints +extern Monitor* SecondaryFreeList_lock; // protects the secondary free region list +extern Mutex* OldSets_lock; // protects the old region sets extern Mutex* MMUTracker_lock; // protects the MMU // tracker data structures extern Mutex* HotCardCache_lock; // protects the hot card cache diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/runtime/relocator.hpp --- a/hotspot/src/share/vm/runtime/relocator.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/runtime/relocator.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -106,7 +106,7 @@ // get the address of in the code_array inline char* addr_at(int bci) const { return (char*) &code_array()[bci]; } - int instruction_length_at(int bci) { return Bytecodes::length_at(code_array() + bci); } + int instruction_length_at(int bci) { return Bytecodes::length_at(NULL, code_array() + bci); } // Helper methods int align(int n) const { return (n+3) & ~3; } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/runtime/sharedRuntime.cpp --- a/hotspot/src/share/vm/runtime/sharedRuntime.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/runtime/sharedRuntime.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -944,9 +944,9 @@ int bci = vfst.bci(); // Find bytecode - Bytecode_invoke* bytecode = Bytecode_invoke_at(caller, bci); - bc = bytecode->java_code(); - int bytecode_index = bytecode->index(); + Bytecode_invoke bytecode(caller, bci); + bc = bytecode.java_code(); + int bytecode_index = bytecode.index(); // Find receiver for non-static call if (bc != Bytecodes::_invokestatic) { @@ -957,7 +957,7 @@ // Caller-frame is a compiled frame frame callerFrame = stubFrame.sender(®_map2); - methodHandle callee = bytecode->static_target(CHECK_(nullHandle)); + methodHandle callee = bytecode.static_target(CHECK_(nullHandle)); if (callee.is_null()) { THROW_(vmSymbols::java_lang_NoSuchMethodException(), nullHandle); } @@ -1674,10 +1674,9 @@ // Get target class name from the checkcast instruction vframeStream vfst(thread, true); assert(!vfst.at_end(), "Java frame must exist"); - Bytecode_checkcast* cc = Bytecode_checkcast_at( - vfst.method()->bcp_from(vfst.bci())); + Bytecode_checkcast cc(vfst.method(), vfst.method()->bcp_from(vfst.bci())); Klass* targetKlass = Klass::cast(vfst.method()->constants()->klass_at( - cc->index(), thread)); + cc.index(), thread)); return generate_class_cast_message(objName, targetKlass->external_name()); } @@ -1711,11 +1710,11 @@ const char* targetType = "the required signature"; vframeStream vfst(thread, true); if (!vfst.at_end()) { - Bytecode_invoke* call = Bytecode_invoke_at(vfst.method(), vfst.bci()); + Bytecode_invoke call(vfst.method(), vfst.bci()); methodHandle target; { EXCEPTION_MARK; - target = call->static_target(THREAD); + target = call.static_target(THREAD); if (HAS_PENDING_EXCEPTION) { CLEAR_PENDING_EXCEPTION; } } if (target.not_null() diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/runtime/thread.cpp --- a/hotspot/src/share/vm/runtime/thread.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/runtime/thread.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -31,6 +31,7 @@ #include "compiler/compileBroker.hpp" #include "interpreter/interpreter.hpp" #include "interpreter/linkResolver.hpp" +#include "jvmtifiles/jvmtiEnv.hpp" #include "memory/oopFactory.hpp" #include "memory/universe.inline.hpp" #include "oops/instanceKlass.hpp" @@ -977,6 +978,19 @@ } #endif // KERNEL +// General purpose hook into Java code, run once when the VM is initialized. +// The Java library method itself may be changed independently from the VM. +static void call_postVMInitHook(TRAPS) { + klassOop k = SystemDictionary::sun_misc_PostVMInitHook_klass(); + instanceKlassHandle klass (THREAD, k); + if (klass.not_null()) { + JavaValue result(T_VOID); + JavaCalls::call_static(&result, klass, vmSymbolHandles::run_method_name(), + vmSymbolHandles::void_method_signature(), + CHECK); + } +} + static void reset_vm_info_property(TRAPS) { // the vm info string ResourceMark rm(THREAD); @@ -1699,7 +1713,7 @@ tlab().make_parsable(true); // retire TLAB } - if (jvmti_thread_state() != NULL) { + if (JvmtiEnv::environments_might_exist()) { JvmtiExport::cleanup_thread(this); } @@ -3345,6 +3359,14 @@ BiasedLocking::init(); + if (JDK_Version::current().post_vm_init_hook_enabled()) { + call_postVMInitHook(THREAD); + // The Java side of PostVMInitHook.run must deal with all + // exceptions and provide means of diagnosis. + if (HAS_PENDING_EXCEPTION) { + CLEAR_PENDING_EXCEPTION; + } + } // Start up the WatcherThread if there are any periodic tasks // NOTE: All PeriodicTasks should be registered by now. If they diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/runtime/thread.hpp --- a/hotspot/src/share/vm/runtime/thread.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/runtime/thread.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -809,7 +809,7 @@ // // _vm_exited is a special value to cover the case of a JavaThread // executing native code after the VM itself is terminated. - TerminatedTypes _terminated; + volatile TerminatedTypes _terminated; // suspend/resume support volatile bool _suspend_equivalent; // Suspend equivalent condition jint _in_deopt_handler; // count of deoptimization diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/runtime/vframeArray.cpp --- a/hotspot/src/share/vm/runtime/vframeArray.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/runtime/vframeArray.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -399,7 +399,7 @@ } else if (TraceDeoptimization) { tty->print(" "); method()->print_value(); - Bytecodes::Code code = Bytecodes::java_code_at(bcp); + Bytecodes::Code code = Bytecodes::java_code_at(method(), bcp); int bci = method()->bci_from(bcp); tty->print(" - %s", Bytecodes::name(code)); tty->print(" @ bci %d ", bci); diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/runtime/vmStructs.cpp --- a/hotspot/src/share/vm/runtime/vmStructs.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/runtime/vmStructs.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -219,8 +219,8 @@ volatile_nonstatic_field(oopDesc, _metadata._compressed_klass, narrowOop) \ static_field(oopDesc, _bs, BarrierSet*) \ nonstatic_field(arrayKlass, _dimension, int) \ - nonstatic_field(arrayKlass, _higher_dimension, klassOop) \ - nonstatic_field(arrayKlass, _lower_dimension, klassOop) \ + volatile_nonstatic_field(arrayKlass, _higher_dimension, klassOop) \ + volatile_nonstatic_field(arrayKlass, _lower_dimension, klassOop) \ nonstatic_field(arrayKlass, _vtable_len, int) \ nonstatic_field(arrayKlass, _alloc_size, juint) \ nonstatic_field(arrayKlass, _component_mirror, oop) \ diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/services/heapDumper.cpp --- a/hotspot/src/share/vm/services/heapDumper.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/services/heapDumper.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -453,7 +453,7 @@ DumpWriter::~DumpWriter() { // flush and close dump file - if (file_descriptor() >= 0) { + if (is_open()) { close(); } if (_buffer != NULL) os::free(_buffer); @@ -463,9 +463,10 @@ // closes dump file (if open) void DumpWriter::close() { // flush and close dump file - if (file_descriptor() >= 0) { + if (is_open()) { flush(); ::close(file_descriptor()); + set_file_descriptor(-1); } } @@ -1935,18 +1936,32 @@ void HeapDumper::dump_heap(bool oome) { static char base_path[JVM_MAXPATHLEN] = {'\0'}; static uint dump_file_seq = 0; - char my_path[JVM_MAXPATHLEN] = {'\0'}; + char* my_path; + const int max_digit_chars = 20; + + const char* dump_file_name = "java_pid"; + const char* dump_file_ext = ".hprof"; // The dump file defaults to java_pid.hprof in the current working // directory. HeapDumpPath= can be used to specify an alternative // dump file name or a directory where dump file is created. if (dump_file_seq == 0) { // first time in, we initialize base_path + // Calculate potentially longest base path and check if we have enough + // allocated statically. + const size_t total_length = + (HeapDumpPath == NULL ? 0 : strlen(HeapDumpPath)) + + strlen(os::file_separator()) + max_digit_chars + + strlen(dump_file_name) + strlen(dump_file_ext) + 1; + if (total_length > sizeof(base_path)) { + warning("Cannot create heap dump file. HeapDumpPath is too long."); + return; + } + bool use_default_filename = true; if (HeapDumpPath == NULL || HeapDumpPath[0] == '\0') { // HeapDumpPath= not specified } else { - assert(strlen(HeapDumpPath) < sizeof(base_path), "HeapDumpPath too long"); - strcpy(base_path, HeapDumpPath); + strncpy(base_path, HeapDumpPath, sizeof(base_path)); // check if the path is a directory (must exist) DIR* dir = os::opendir(base_path); if (dir == NULL) { @@ -1960,8 +1975,6 @@ char* end = base_path; end += (strlen(base_path) - fs_len); if (strcmp(end, os::file_separator()) != 0) { - assert(strlen(base_path) + strlen(os::file_separator()) < sizeof(base_path), - "HeapDumpPath too long"); strcat(base_path, os::file_separator()); } } @@ -1969,21 +1982,26 @@ } // If HeapDumpPath wasn't a file name then we append the default name if (use_default_filename) { - char fn[32]; - sprintf(fn, "java_pid%d", os::current_process_id()); - assert(strlen(base_path) + strlen(fn) + strlen(".hprof") < sizeof(base_path), "HeapDumpPath too long"); - strcat(base_path, fn); - strcat(base_path, ".hprof"); + const size_t dlen = strlen(base_path); // if heap dump dir specified + jio_snprintf(&base_path[dlen], sizeof(base_path)-dlen, "%s%d%s", + dump_file_name, os::current_process_id(), dump_file_ext); } - assert(strlen(base_path) < sizeof(my_path), "Buffer too small"); - strcpy(my_path, base_path); + const size_t len = strlen(base_path) + 1; + my_path = (char*)os::malloc(len); + if (my_path == NULL) { + warning("Cannot create heap dump file. Out of system memory."); + return; + } + strncpy(my_path, base_path, len); } else { // Append a sequence number id for dumps following the first - char fn[33]; - sprintf(fn, ".%d", dump_file_seq); - assert(strlen(base_path) + strlen(fn) < sizeof(my_path), "HeapDumpPath too long"); - strcpy(my_path, base_path); - strcat(my_path, fn); + const size_t len = strlen(base_path) + max_digit_chars + 2; // for '.' and \0 + my_path = (char*)os::malloc(len); + if (my_path == NULL) { + warning("Cannot create heap dump file. Out of system memory."); + return; + } + jio_snprintf(my_path, len, "%s.%d", base_path, dump_file_seq); } dump_file_seq++; // increment seq number for next time we dump @@ -1991,4 +2009,5 @@ true /* send to tty */, oome /* pass along out-of-memory-error flag */); dumper.dump(my_path); + os::free(my_path); } diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/utilities/debug.hpp --- a/hotspot/src/share/vm/utilities/debug.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/utilities/debug.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -34,6 +34,7 @@ class FormatBuffer { public: inline FormatBuffer(const char * format, ...); + inline void append(const char* format, ...); operator const char *() const { return _buf; } private: @@ -51,6 +52,19 @@ va_end(argp); } +template +void FormatBuffer::append(const char* format, ...) { + // Given that the constructor does a vsnprintf we can assume that + // _buf is already initialized. + size_t len = strlen(_buf); + char* buf_end = _buf + len; + + va_list argp; + va_start(argp, format); + vsnprintf(buf_end, bufsz - len, format, argp); + va_end(argp); +} + // Used to format messages for assert(), guarantee(), fatal(), etc. typedef FormatBuffer<> err_msg; diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/utilities/globalDefinitions.hpp --- a/hotspot/src/share/vm/utilities/globalDefinitions.hpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/utilities/globalDefinitions.hpp Tue Feb 01 18:57:53 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1179,6 +1179,8 @@ // '%d' formats to indicate a 64-bit quantity; commonly "l" (in LP64) or "ll" // (in ILP32). +#define BOOL_TO_STR(__b) (__b) ? "true" : "false" + // Format 32-bit quantities. #define INT32_FORMAT "%d" #define UINT32_FORMAT "%u" diff -r 25b4fa8c7dd2 -r 5eecea832c83 hotspot/src/share/vm/utilities/vmError.cpp --- a/hotspot/src/share/vm/utilities/vmError.cpp Thu Jan 27 17:28:34 2011 -0800 +++ b/hotspot/src/share/vm/utilities/vmError.cpp Tue Feb 01 18:57:53 2011 -0800 @@ -874,11 +874,13 @@ } if (fd == -1) { - // try temp directory const char * tmpdir = os::get_temp_directory(); - jio_snprintf(buffer, sizeof(buffer), "%s%shs_err_pid%u.log", - tmpdir, os::file_separator(), os::current_process_id()); - fd = open(buffer, O_WRONLY | O_CREAT | O_TRUNC, 0666); + // try temp directory if it exists. + if (tmpdir != NULL && tmpdir[0] != '\0') { + jio_snprintf(buffer, sizeof(buffer), "%s%shs_err_pid%u.log", + tmpdir, os::file_separator(), os::current_process_id()); + fd = open(buffer, O_WRONLY | O_CREAT | O_TRUNC, 0666); + } } if (fd != -1) {