821 else |
821 else |
822 __ brx(Assembler::greaterEqualUnsigned, false, Assembler::pt, (*NOLp)); |
822 __ brx(Assembler::greaterEqualUnsigned, false, Assembler::pt, (*NOLp)); |
823 __ delayed()->nop(); |
823 __ delayed()->nop(); |
824 } |
824 } |
825 |
825 |
826 // |
|
827 // Generate pre-write barrier for array. |
|
828 // |
|
829 // Input: |
|
830 // addr - register containing starting address |
|
831 // count - register containing element count |
|
832 // tmp - scratch register |
|
833 // |
|
834 // The input registers are overwritten. |
|
835 // |
|
836 void gen_write_ref_array_pre_barrier(Register addr, Register count, bool dest_uninitialized) { |
|
837 BarrierSet* bs = Universe::heap()->barrier_set(); |
|
838 switch (bs->kind()) { |
|
839 case BarrierSet::G1BarrierSet: |
|
840 // With G1, don't generate the call if we statically know that the target in uninitialized |
|
841 if (!dest_uninitialized) { |
|
842 Register tmp = O5; |
|
843 assert_different_registers(addr, count, tmp); |
|
844 Label filtered; |
|
845 // Is marking active? |
|
846 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { |
|
847 __ ld(G2, in_bytes(JavaThread::satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_active()), tmp); |
|
848 } else { |
|
849 guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, |
|
850 "Assumption"); |
|
851 __ ldsb(G2, in_bytes(JavaThread::satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_active()), tmp); |
|
852 } |
|
853 // Is marking active? |
|
854 __ cmp_and_br_short(tmp, G0, Assembler::equal, Assembler::pt, filtered); |
|
855 |
|
856 __ save_frame(0); |
|
857 // Save the necessary global regs... will be used after. |
|
858 if (addr->is_global()) { |
|
859 __ mov(addr, L0); |
|
860 } |
|
861 if (count->is_global()) { |
|
862 __ mov(count, L1); |
|
863 } |
|
864 __ mov(addr->after_save(), O0); |
|
865 // Get the count into O1 |
|
866 __ call(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre)); |
|
867 __ delayed()->mov(count->after_save(), O1); |
|
868 if (addr->is_global()) { |
|
869 __ mov(L0, addr); |
|
870 } |
|
871 if (count->is_global()) { |
|
872 __ mov(L1, count); |
|
873 } |
|
874 __ restore(); |
|
875 |
|
876 __ bind(filtered); |
|
877 DEBUG_ONLY(__ set(0xDEADC0DE, tmp);) // we have killed tmp |
|
878 } |
|
879 break; |
|
880 case BarrierSet::CardTableBarrierSet: |
|
881 break; |
|
882 default: |
|
883 ShouldNotReachHere(); |
|
884 } |
|
885 } |
|
886 // |
|
887 // Generate post-write barrier for array. |
|
888 // |
|
889 // Input: |
|
890 // addr - register containing starting address |
|
891 // count - register containing element count |
|
892 // tmp - scratch register |
|
893 // |
|
894 // The input registers are overwritten. |
|
895 // |
|
896 void gen_write_ref_array_post_barrier(Register addr, Register count, |
|
897 Register tmp) { |
|
898 BarrierSet* bs = Universe::heap()->barrier_set(); |
|
899 |
|
900 switch (bs->kind()) { |
|
901 case BarrierSet::G1BarrierSet: |
|
902 { |
|
903 // Get some new fresh output registers. |
|
904 __ save_frame(0); |
|
905 __ mov(addr->after_save(), O0); |
|
906 __ call(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post)); |
|
907 __ delayed()->mov(count->after_save(), O1); |
|
908 __ restore(); |
|
909 } |
|
910 break; |
|
911 case BarrierSet::CardTableBarrierSet: |
|
912 { |
|
913 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs); |
|
914 CardTable* ct = ctbs->card_table(); |
|
915 assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code"); |
|
916 assert_different_registers(addr, count, tmp); |
|
917 |
|
918 Label L_loop, L_done; |
|
919 |
|
920 __ cmp_and_br_short(count, 0, Assembler::equal, Assembler::pt, L_done); // zero count - nothing to do |
|
921 |
|
922 __ sll_ptr(count, LogBytesPerHeapOop, count); |
|
923 __ sub(count, BytesPerHeapOop, count); |
|
924 __ add(count, addr, count); |
|
925 // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.) |
|
926 __ srl_ptr(addr, CardTable::card_shift, addr); |
|
927 __ srl_ptr(count, CardTable::card_shift, count); |
|
928 __ sub(count, addr, count); |
|
929 AddressLiteral rs(ct->byte_map_base()); |
|
930 __ set(rs, tmp); |
|
931 __ BIND(L_loop); |
|
932 __ stb(G0, tmp, addr); |
|
933 __ subcc(count, 1, count); |
|
934 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_loop); |
|
935 __ delayed()->add(addr, 1, addr); |
|
936 __ BIND(L_done); |
|
937 } |
|
938 break; |
|
939 case BarrierSet::ModRef: |
|
940 break; |
|
941 default: |
|
942 ShouldNotReachHere(); |
|
943 } |
|
944 } |
|
945 |
826 |
946 // |
827 // |
947 // Generate main code for disjoint arraycopy |
828 // Generate main code for disjoint arraycopy |
948 // |
829 // |
949 typedef void (StubGenerator::*CopyLoopFunc)(Register from, Register to, Register count, int count_dec, |
830 typedef void (StubGenerator::*CopyLoopFunc)(Register from, Register to, Register count, int count_dec, |
2386 *entry = __ pc(); |
2267 *entry = __ pc(); |
2387 // caller can pass a 64-bit byte count here |
2268 // caller can pass a 64-bit byte count here |
2388 BLOCK_COMMENT("Entry:"); |
2269 BLOCK_COMMENT("Entry:"); |
2389 } |
2270 } |
2390 |
2271 |
2391 // save arguments for barrier generation |
2272 DecoratorSet decorators = ARRAYCOPY_DISJOINT; |
2392 __ mov(to, G1); |
2273 if (dest_uninitialized) { |
2393 __ mov(count, G5); |
2274 decorators |= AS_DEST_NOT_INITIALIZED; |
2394 gen_write_ref_array_pre_barrier(G1, G5, dest_uninitialized); |
2275 } |
|
2276 if (aligned) { |
|
2277 decorators |= ARRAYCOPY_ALIGNED; |
|
2278 } |
|
2279 |
|
2280 BarrierSetAssembler *bs = Universe::heap()->barrier_set()->barrier_set_assembler(); |
|
2281 bs->arraycopy_prologue(_masm, decorators, T_OBJECT, from, to, count); |
|
2282 |
2395 assert_clean_int(count, O3); // Make sure 'count' is clean int. |
2283 assert_clean_int(count, O3); // Make sure 'count' is clean int. |
2396 if (UseCompressedOops) { |
2284 if (UseCompressedOops) { |
2397 generate_disjoint_int_copy_core(aligned); |
2285 generate_disjoint_int_copy_core(aligned); |
2398 } else { |
2286 } else { |
2399 generate_disjoint_long_copy_core(aligned); |
2287 generate_disjoint_long_copy_core(aligned); |
2400 } |
2288 } |
2401 // O0 is used as temp register |
2289 |
2402 gen_write_ref_array_post_barrier(G1, G5, O0); |
2290 bs->arraycopy_epilogue(_masm, decorators, T_OBJECT, from, to, count); |
2403 |
2291 |
2404 // O3, O4 are used as temp registers |
2292 // O3, O4 are used as temp registers |
2405 inc_counter_np(SharedRuntime::_oop_array_copy_ctr, O3, O4); |
2293 inc_counter_np(SharedRuntime::_oop_array_copy_ctr, O3, O4); |
2406 __ retl(); |
2294 __ retl(); |
2407 __ delayed()->mov(G0, O0); // return 0 |
2295 __ delayed()->mov(G0, O0); // return 0 |
2436 BLOCK_COMMENT("Entry:"); |
2324 BLOCK_COMMENT("Entry:"); |
2437 } |
2325 } |
2438 |
2326 |
2439 array_overlap_test(nooverlap_target, LogBytesPerHeapOop); |
2327 array_overlap_test(nooverlap_target, LogBytesPerHeapOop); |
2440 |
2328 |
2441 // save arguments for barrier generation |
2329 DecoratorSet decorators = 0; |
2442 __ mov(to, G1); |
2330 if (dest_uninitialized) { |
2443 __ mov(count, G5); |
2331 decorators |= AS_DEST_NOT_INITIALIZED; |
2444 gen_write_ref_array_pre_barrier(G1, G5, dest_uninitialized); |
2332 } |
|
2333 if (aligned) { |
|
2334 decorators |= ARRAYCOPY_ALIGNED; |
|
2335 } |
|
2336 |
|
2337 BarrierSetAssembler *bs = Universe::heap()->barrier_set()->barrier_set_assembler(); |
|
2338 bs->arraycopy_prologue(_masm, decorators, T_OBJECT, from, to, count); |
2445 |
2339 |
2446 if (UseCompressedOops) { |
2340 if (UseCompressedOops) { |
2447 generate_conjoint_int_copy_core(aligned); |
2341 generate_conjoint_int_copy_core(aligned); |
2448 } else { |
2342 } else { |
2449 generate_conjoint_long_copy_core(aligned); |
2343 generate_conjoint_long_copy_core(aligned); |
2450 } |
2344 } |
2451 |
2345 |
2452 // O0 is used as temp register |
2346 bs->arraycopy_epilogue(_masm, decorators, T_OBJECT, from, to, count); |
2453 gen_write_ref_array_post_barrier(G1, G5, O0); |
|
2454 |
2347 |
2455 // O3, O4 are used as temp registers |
2348 // O3, O4 are used as temp registers |
2456 inc_counter_np(SharedRuntime::_oop_array_copy_ctr, O3, O4); |
2349 inc_counter_np(SharedRuntime::_oop_array_copy_ctr, O3, O4); |
2457 __ retl(); |
2350 __ retl(); |
2458 __ delayed()->mov(G0, O0); // return 0 |
2351 __ delayed()->mov(G0, O0); // return 0 |
2550 if (entry != NULL) { |
2443 if (entry != NULL) { |
2551 *entry = __ pc(); |
2444 *entry = __ pc(); |
2552 // caller can pass a 64-bit byte count here (from generic stub) |
2445 // caller can pass a 64-bit byte count here (from generic stub) |
2553 BLOCK_COMMENT("Entry:"); |
2446 BLOCK_COMMENT("Entry:"); |
2554 } |
2447 } |
2555 gen_write_ref_array_pre_barrier(O1_to, O2_count, dest_uninitialized); |
2448 |
2556 |
2449 DecoratorSet decorators = ARRAYCOPY_CHECKCAST; |
2557 Label load_element, store_element, do_card_marks, fail, done; |
2450 if (dest_uninitialized) { |
|
2451 decorators |= AS_DEST_NOT_INITIALIZED; |
|
2452 } |
|
2453 |
|
2454 BarrierSetAssembler *bs = Universe::heap()->barrier_set()->barrier_set_assembler(); |
|
2455 bs->arraycopy_prologue(_masm, decorators, T_OBJECT, O0_from, O1_to, O2_count); |
|
2456 |
|
2457 Label load_element, store_element, do_epilogue, fail, done; |
2558 __ addcc(O2_count, 0, G1_remain); // initialize loop index, and test it |
2458 __ addcc(O2_count, 0, G1_remain); // initialize loop index, and test it |
2559 __ brx(Assembler::notZero, false, Assembler::pt, load_element); |
2459 __ brx(Assembler::notZero, false, Assembler::pt, load_element); |
2560 __ delayed()->mov(G0, O5_offset); // offset from start of arrays |
2460 __ delayed()->mov(G0, O5_offset); // offset from start of arrays |
2561 |
2461 |
2562 // Empty array: Nothing to do. |
2462 // Empty array: Nothing to do. |
2574 |
2474 |
2575 __ BIND(store_element); |
2475 __ BIND(store_element); |
2576 __ deccc(G1_remain); // decrement the count |
2476 __ deccc(G1_remain); // decrement the count |
2577 __ store_heap_oop(G3_oop, O1_to, O5_offset); // store the oop |
2477 __ store_heap_oop(G3_oop, O1_to, O5_offset); // store the oop |
2578 __ inc(O5_offset, heapOopSize); // step to next offset |
2478 __ inc(O5_offset, heapOopSize); // step to next offset |
2579 __ brx(Assembler::zero, true, Assembler::pt, do_card_marks); |
2479 __ brx(Assembler::zero, true, Assembler::pt, do_epilogue); |
2580 __ delayed()->set(0, O0); // return -1 on success |
2480 __ delayed()->set(0, O0); // return -1 on success |
2581 |
2481 |
2582 // ======== loop entry is here ======== |
2482 // ======== loop entry is here ======== |
2583 __ BIND(load_element); |
2483 __ BIND(load_element); |
2584 __ load_heap_oop(O0_from, O5_offset, G3_oop); // load the oop |
2484 __ load_heap_oop(O0_from, O5_offset, G3_oop); // load the oop |
2598 __ BIND(fail); |
2498 __ BIND(fail); |
2599 __ subcc(O2_count, G1_remain, O2_count); |
2499 __ subcc(O2_count, G1_remain, O2_count); |
2600 __ brx(Assembler::zero, false, Assembler::pt, done); |
2500 __ brx(Assembler::zero, false, Assembler::pt, done); |
2601 __ delayed()->not1(O2_count, O0); // report (-1^K) to caller |
2501 __ delayed()->not1(O2_count, O0); // report (-1^K) to caller |
2602 |
2502 |
2603 __ BIND(do_card_marks); |
2503 __ BIND(do_epilogue); |
2604 gen_write_ref_array_post_barrier(O1_to, O2_count, O3); // store check on O1[0..O2] |
2504 bs->arraycopy_epilogue(_masm, decorators, T_OBJECT, O0_from, O1_to, O2_count); |
2605 |
2505 |
2606 __ BIND(done); |
2506 __ BIND(done); |
2607 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr, O3, O4); |
2507 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr, O3, O4); |
2608 __ retl(); |
2508 __ retl(); |
2609 __ delayed()->nop(); // return value in 00 |
2509 __ delayed()->nop(); // return value in 00 |