# HG changeset patch # User zgu # Date 1556804992 14400 # Node ID df2b3565f343adced309bfe8399e4b6e71b4d651 # Parent 09f09b4e78089b27b052dce530dcd36b016bb721 8222992: Shenandoah: Pre-evacuate all roots Reviewed-by: shade diff -r 09f09b4e7808 -r df2b3565f343 src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp Thu May 02 06:33:28 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp Thu May 02 09:49:52 2019 -0400 @@ -79,4 +79,16 @@ inline void do_oop_work(T* p); }; +#ifdef ASSERT +class ShenandoahAssertNotForwardedClosure : public OopClosure { +private: + template + inline void do_oop_work(T* p); + +public: + inline void do_oop(narrowOop* p); + inline void do_oop(oop* p); +}; +#endif // ASSERT + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCLOSURES_HPP diff -r 09f09b4e7808 -r df2b3565f343 src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp Thu May 02 06:33:28 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp Thu May 02 09:49:52 2019 -0400 @@ -107,4 +107,18 @@ do_oop_work(p); } +#ifdef ASSERT +template +void ShenandoahAssertNotForwardedClosure::do_oop_work(T* p) { + T o = RawAccess<>::oop_load(p); + if (!CompressedOops::is_null(o)) { + oop obj = CompressedOops::decode_not_null(o); + shenandoah_assert_not_forwarded(p, obj); + } +} + +void ShenandoahAssertNotForwardedClosure::do_oop(narrowOop* p) { do_oop_work(p); } +void ShenandoahAssertNotForwardedClosure::do_oop(oop* p) { do_oop_work(p); } +#endif + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCLOSURES_HPP diff -r 09f09b4e7808 -r df2b3565f343 src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp Thu May 02 06:33:28 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp Thu May 02 09:49:52 2019 -0400 @@ -661,15 +661,13 @@ // anything to them. void ShenandoahConcurrentMark::weak_roots_work() { WorkGang* workers = _heap->workers(); - ShenandoahIsAliveSelector is_alive; - - if (_heap->has_forwarded_objects()) { - ShenandoahWeakUpdateClosure cl; - WeakProcessor::weak_oops_do(workers, is_alive.is_alive_closure(), &cl, 1); - } else { - ShenandoahWeakAssertNotForwardedClosure cl; - WeakProcessor::weak_oops_do(workers, is_alive.is_alive_closure(), &cl, 1); - } + OopClosure* keep_alive = &do_nothing_cl; +#ifdef ASSERT + ShenandoahWeakAssertNotForwardedClosure verify_cl; + keep_alive = &verify_cl; +#endif + ShenandoahIsAliveClosure is_alive; + WeakProcessor::weak_oops_do(workers, &is_alive, keep_alive, 1); } void ShenandoahConcurrentMark::weak_refs_work_doit(bool full_gc) { diff -r 09f09b4e7808 -r df2b3565f343 src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp Thu May 02 06:33:28 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp Thu May 02 09:49:52 2019 -0400 @@ -1487,8 +1487,13 @@ if (!cancelled_gc()) { concurrent_mark()->finish_mark_from_roots(/* full_gc = */ false); - if (has_forwarded_objects()) { - concurrent_mark()->update_roots(ShenandoahPhaseTimings::update_roots); + // Degen may be caused by failed evacuation of roots + if (is_degenerated_gc_in_progress() && has_forwarded_objects()) { + concurrent_mark()->update_roots(ShenandoahPhaseTimings::degen_gc_update_roots); + } + + if (ShenandoahVerify) { + verifier()->verify_roots_no_forwarded(); } stop_concurrent_marking(); @@ -1540,6 +1545,7 @@ } if (ShenandoahVerify) { + verifier()->verify_roots_no_forwarded(); verifier()->verify_during_evacuation(); } } else { @@ -2176,8 +2182,8 @@ assert(!cancelled_gc(), "Should have been done right before"); concurrent_mark()->update_roots(is_degenerated_gc_in_progress() ? - ShenandoahPhaseTimings::degen_gc_update_roots: - ShenandoahPhaseTimings::final_update_refs_roots); + ShenandoahPhaseTimings::degen_gc_update_roots: + ShenandoahPhaseTimings::final_update_refs_roots); ShenandoahGCPhase final_update_refs(ShenandoahPhaseTimings::final_update_refs_recycle); @@ -2186,6 +2192,7 @@ set_update_refs_in_progress(false); if (ShenandoahVerify) { + verifier()->verify_roots_no_forwarded(); verifier()->verify_after_updaterefs(); } diff -r 09f09b4e7808 -r df2b3565f343 src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp Thu May 02 06:33:28 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp Thu May 02 09:49:52 2019 -0400 @@ -42,26 +42,6 @@ #include "runtime/thread.hpp" #include "services/management.hpp" -struct PhaseMap { - WeakProcessorPhases::Phase _weak_processor_phase; - ShenandoahPhaseTimings::GCParPhases _shenandoah_phase; -}; - -static const struct PhaseMap phase_mapping[] = { -#if INCLUDE_JVMTI - {WeakProcessorPhases::jvmti, ShenandoahPhaseTimings::JVMTIWeakRoots}, -#endif -#if INCLUDE_JFR - {WeakProcessorPhases::jfr, ShenandoahPhaseTimings::JFRWeakRoots}, -#endif - {WeakProcessorPhases::jni, ShenandoahPhaseTimings::JNIWeakRoots}, - {WeakProcessorPhases::stringtable, ShenandoahPhaseTimings::StringTableRoots}, - {WeakProcessorPhases::resolved_method_table, ShenandoahPhaseTimings::ResolvedMethodTableRoots}, - {WeakProcessorPhases::vm, ShenandoahPhaseTimings::VMWeakRoots} -}; - -STATIC_ASSERT(sizeof(phase_mapping) / sizeof(PhaseMap) == WeakProcessorPhases::phase_count); - ShenandoahRootProcessor::ShenandoahRootProcessor(ShenandoahHeap* heap, uint n_workers, ShenandoahPhaseTimings::Phase phase) : _process_strong_tasks(new SubTasksDone(SHENANDOAH_RP_PS_NumElements)), @@ -84,32 +64,16 @@ StringDedup::gc_epilogue(); } - ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times(); - if (_processed_weak_roots) { assert(_weak_processor_timings.max_threads() == n_workers(), "Must match"); - for (uint index = 0; index < WeakProcessorPhases::phase_count; index ++) { - weak_processor_timing_to_shenandoah_timing(phase_mapping[index]._weak_processor_phase, - phase_mapping[index]._shenandoah_phase, - worker_times); - } + ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times(); + ShenandoahTimingConverter::weak_processing_timing_to_shenandoah_timing(&_weak_processor_timings, + worker_times); } ShenandoahHeap::heap()->phase_timings()->record_workers_end(_phase); } -void ShenandoahRootProcessor::weak_processor_timing_to_shenandoah_timing(const WeakProcessorPhases::Phase wpp, - const ShenandoahPhaseTimings::GCParPhases spp, - ShenandoahWorkerTimings* worker_times) const { - if (WeakProcessorPhases::is_serial(wpp)) { - worker_times->record_time_secs(spp, 0, _weak_processor_timings.phase_time_sec(wpp)); - } else { - for (uint index = 0; index < _weak_processor_timings.max_threads(); index ++) { - worker_times->record_time_secs(spp, index, _weak_processor_timings.worker_time_sec(index, wpp)); - } - } -} - void ShenandoahRootProcessor::process_all_roots_slow(OopClosure* oops) { CLDToOopClosure clds(oops, ClassLoaderData::_claim_strong); CodeBlobToOopClosure blobs(oops, !CodeBlobToOopClosure::FixRelocations); @@ -251,7 +215,9 @@ _evacuation_tasks(new SubTasksDone(SHENANDOAH_EVAC_NumElements)), _srs(n_workers), _phase(phase), - _coderoots_cset_iterator(ShenandoahCodeRoots::cset_iterator()) { + _coderoots_cset_iterator(ShenandoahCodeRoots::cset_iterator()), + _weak_processor_timings(n_workers), + _weak_processor_task(&_weak_processor_timings, n_workers) { heap->phase_timings()->record_workers_start(_phase); if (ShenandoahStringDedup::is_enabled()) { StringDedup::gc_prologue(false); @@ -263,6 +229,12 @@ if (ShenandoahStringDedup::is_enabled()) { StringDedup::gc_epilogue(); } + + ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times(); + assert(_weak_processor_timings.max_threads() == n_workers(), "Must match"); + ShenandoahTimingConverter::weak_processing_timing_to_shenandoah_timing(&_weak_processor_timings, + worker_times); + ShenandoahHeap::heap()->phase_timings()->record_workers_end(_phase); } @@ -270,13 +242,19 @@ CodeBlobClosure* blobs, uint worker_id) { + AlwaysTrueClosure always_true; ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times(); { bool is_par = n_workers() > 1; ResourceMark rm; ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::ThreadRoots, worker_id); + Threads::possibly_parallel_oops_do(is_par, oops, NULL); + } - Threads::possibly_parallel_oops_do(is_par, oops, NULL); + { + ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::CLDGRoots, worker_id); + CLDToOopClosure clds(oops, ClassLoaderData::_claim_strong); + _cld_iterator.root_cld_do(&clds, &clds); } if (blobs != NULL) { @@ -285,8 +263,7 @@ } if (ShenandoahStringDedup::is_enabled()) { - ShenandoahForwardedIsAliveClosure is_alive; - ShenandoahStringDedup::parallel_oops_do(&is_alive, oops, worker_id); + ShenandoahStringDedup::parallel_oops_do(&always_true, oops, worker_id); } if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_Universe_oops_do)) { @@ -306,6 +283,11 @@ JvmtiExport::weak_oops_do(&is_alive, oops); } + if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_JNIHandles_oops_do)) { + ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::JNIRoots, worker_id); + JNIHandles::oops_do(oops); + } + if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_SystemDictionary_oops_do)) { ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::SystemDictionaryRoots, worker_id); SystemDictionary::oops_do(oops); @@ -316,6 +298,7 @@ ObjectSynchronizer::oops_do(oops); } + _weak_processor_task.work(worker_id, &always_true, oops); } uint ShenandoahRootEvacuator::n_workers() const { diff -r 09f09b4e7808 -r df2b3565f343 src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp Thu May 02 06:33:28 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp Thu May 02 09:49:52 2019 -0400 @@ -119,6 +119,9 @@ StrongRootsScope _srs; ShenandoahPhaseTimings::Phase _phase; ShenandoahCsetCodeRootsIterator _coderoots_cset_iterator; + ParallelCLDRootIterator _cld_iterator; + WeakProcessorPhaseTimes _weak_processor_timings; + WeakProcessor::Task _weak_processor_task; enum Shenandoah_evacuate_roots_tasks { SHENANDOAH_EVAC_Universe_oops_do, @@ -126,6 +129,7 @@ SHENANDOAH_EVAC_Management_oops_do, SHENANDOAH_EVAC_SystemDictionary_oops_do, SHENANDOAH_EVAC_jvmti_oops_do, + SHENANDOAH_EVAC_JNIHandles_oops_do, // Leave this one last. SHENANDOAH_EVAC_NumElements }; diff -r 09f09b4e7808 -r df2b3565f343 src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp Thu May 02 06:33:28 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp Thu May 02 09:49:52 2019 -0400 @@ -183,3 +183,47 @@ ShenandoahThreadLocalData::set_worker_id(thr, ShenandoahThreadLocalData::INVALID_WORKER_ID); #endif } + +struct PhaseMap { + WeakProcessorPhases::Phase _weak_processor_phase; + ShenandoahPhaseTimings::GCParPhases _shenandoah_phase; +}; + +static const struct PhaseMap phase_mapping[] = { +#if INCLUDE_JVMTI + {WeakProcessorPhases::jvmti, ShenandoahPhaseTimings::JVMTIWeakRoots}, +#endif +#if INCLUDE_JFR + {WeakProcessorPhases::jfr, ShenandoahPhaseTimings::JFRWeakRoots}, +#endif + {WeakProcessorPhases::jni, ShenandoahPhaseTimings::JNIWeakRoots}, + {WeakProcessorPhases::stringtable, ShenandoahPhaseTimings::StringTableRoots}, + {WeakProcessorPhases::resolved_method_table, ShenandoahPhaseTimings::ResolvedMethodTableRoots}, + {WeakProcessorPhases::vm, ShenandoahPhaseTimings::VMWeakRoots} +}; + +STATIC_ASSERT(sizeof(phase_mapping) / sizeof(PhaseMap) == WeakProcessorPhases::phase_count); + +void ShenandoahTimingConverter::weak_processing_timing_to_shenandoah_timing(WeakProcessorPhaseTimes* weak_processing_timings, + ShenandoahWorkerTimings* sh_worker_times) { + assert(weak_processing_timings->max_threads() == weak_processing_timings->max_threads(), "Must match"); + for (uint index = 0; index < WeakProcessorPhases::phase_count; index ++) { + weak_processing_phase_to_shenandoah_phase(phase_mapping[index]._weak_processor_phase, + weak_processing_timings, + phase_mapping[index]._shenandoah_phase, + sh_worker_times); + } +} + +void ShenandoahTimingConverter::weak_processing_phase_to_shenandoah_phase(WeakProcessorPhases::Phase wpp, + WeakProcessorPhaseTimes* weak_processing_timings, + ShenandoahPhaseTimings::GCParPhases spp, + ShenandoahWorkerTimings* sh_worker_times) { + if (WeakProcessorPhases::is_serial(wpp)) { + sh_worker_times->record_time_secs(spp, 0, weak_processing_timings->phase_time_sec(wpp)); + } else { + for (uint index = 0; index < weak_processing_timings->max_threads(); index ++) { + sh_worker_times->record_time_secs(spp, index, weak_processing_timings->worker_time_sec(index, wpp)); + } + } +} diff -r 09f09b4e7808 -r df2b3565f343 src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp Thu May 02 06:33:28 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp Thu May 02 09:49:52 2019 -0400 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * Copyright (c) 2017, 2019, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -24,14 +24,14 @@ #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHUTILS_HPP #define SHARE_GC_SHENANDOAH_SHENANDOAHUTILS_HPP -#include "jfr/jfrEvents.hpp" - #include "gc/shared/gcCause.hpp" +#include "gc/shared/gcVMOperations.hpp" #include "gc/shared/isGCActiveMark.hpp" #include "gc/shared/suspendibleThreadSet.hpp" -#include "gc/shared/gcVMOperations.hpp" +#include "gc/shared/weakProcessorPhaseTimes.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" #include "gc/shenandoah/shenandoahThreadLocalData.hpp" +#include "jfr/jfrEvents.hpp" #include "memory/allocation.hpp" #include "runtime/safepoint.hpp" #include "runtime/vmThread.hpp" @@ -175,4 +175,15 @@ } }; +class ShenandoahTimingConverter : public AllStatic { +public: + static void weak_processing_timing_to_shenandoah_timing(WeakProcessorPhaseTimes* weak_processing_timings, + ShenandoahWorkerTimings* sh_worker_times); +private: + static void weak_processing_phase_to_shenandoah_phase(WeakProcessorPhases::Phase wpp, + WeakProcessorPhaseTimes* weak_processing_timings, + ShenandoahPhaseTimings::GCParPhases spp, + ShenandoahWorkerTimings* sh_worker_times); +}; + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHUTILS_HPP diff -r 09f09b4e7808 -r df2b3565f343 src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp Thu May 02 06:33:28 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp Thu May 02 09:49:52 2019 -0400 @@ -919,3 +919,31 @@ _verify_gcstate_stable // full gc cleaned up everything ); } + +class ShenandoahVerifyNoForwared : public OopClosure { +private: + template + void do_oop_work(T* p) { + T o = RawAccess<>::oop_load(p); + if (!CompressedOops::is_null(o)) { + oop obj = CompressedOops::decode_not_null(o); + oop fwd = (oop) ShenandoahBrooksPointer::get_raw_unchecked(obj); + if (!oopDesc::equals_raw(obj, fwd)) { + ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL, + "Verify Roots", "Should not be forwarded", __FILE__, __LINE__); + } + } + } + +public: + void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p); } +}; + +void ShenandoahVerifier::verify_roots_no_forwarded() { + guarantee(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "only when nothing else happens"); + ShenandoahRootProcessor rp(_heap, 1, ShenandoahPhaseTimings::_num_phases); // no need for stats + ShenandoahVerifyNoForwared cl; + rp.process_all_roots_slow(&cl); +} + diff -r 09f09b4e7808 -r df2b3565f343 src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp Thu May 02 06:33:28 2019 -0700 +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp Thu May 02 09:49:52 2019 -0400 @@ -186,6 +186,9 @@ void verify_after_traversal(); void verify_after_degenerated(); void verify_generic(VerifyOption option); + + // Roots should only contain to-space oops + void verify_roots_no_forwarded(); }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHVERIFIER_HPP