# HG changeset patch # User kvn # Date 1501874673 25200 # Node ID b9a409ea423bd266af5c3e3a1292fef61e36649d # Parent 2e49bb347cbcae82f5161af56436030524332312# Parent 137ae24f3b528bdb08c10294b96e1feb137b3cef Merge diff -r 2e49bb347cbc -r b9a409ea423b hotspot/src/os_cpu/linux_zero/vm/os_linux_zero.hpp --- a/hotspot/src/os_cpu/linux_zero/vm/os_linux_zero.hpp Mon Jul 03 14:33:00 2017 +0200 +++ b/hotspot/src/os_cpu/linux_zero/vm/os_linux_zero.hpp Fri Aug 04 12:24:33 2017 -0700 @@ -38,10 +38,10 @@ static void atomic_copy64(const volatile void *src, volatile void *dst) { #if defined(PPC32) double tmp; - asm volatile ("lfd %0, 0(%1)\n" - "stfd %0, 0(%2)\n" - : "=f"(tmp) - : "b"(src), "b"(dst)); + asm volatile ("lfd %0, %2\n" + "stfd %0, %1\n" + : "=&f"(tmp), "=Q"(*(volatile double*)dst) + : "Q"(*(volatile double*)src)); #elif defined(S390) && !defined(_LP64) double tmp; asm volatile ("ld %0, 0(%1)\n" diff -r 2e49bb347cbc -r b9a409ea423b hotspot/src/share/tools/hsdis/Makefile --- a/hotspot/src/share/tools/hsdis/Makefile Mon Jul 03 14:33:00 2017 +0200 +++ b/hotspot/src/share/tools/hsdis/Makefile Fri Aug 04 12:24:33 2017 -0700 @@ -1,5 +1,5 @@ # -# Copyright (c) 2008, 2013, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -70,14 +70,18 @@ else #linux CPU = $(shell uname -m) ARCH1=$(CPU:x86_64=amd64) -ARCH=$(ARCH1:i686=i386) +ARCH2=$(ARCH1:i686=i386) +ARCH=$(ARCH2:sparc64=sparcv9) ifdef LP64 CFLAGS/sparcv9 += -m64 CFLAGS/amd64 += -m64 CFLAGS/ppc64 += -m64 CFLAGS/ppc64le += -m64 -DABI_ELFv2 else -ARCH=$(ARCH1:amd64=i386) +ARCH=$(ARCH2:amd64=i386) +ifneq ($(findstring arm,$(ARCH)),) +ARCH=arm +endif CFLAGS/i386 += -m32 CFLAGS/sparc += -m32 endif diff -r 2e49bb347cbc -r b9a409ea423b hotspot/src/share/tools/hsdis/hsdis.c --- a/hotspot/src/share/tools/hsdis/hsdis.c Mon Jul 03 14:33:00 2017 +0200 +++ b/hotspot/src/share/tools/hsdis/hsdis.c Fri Aug 04 12:24:33 2017 -0700 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -465,6 +465,9 @@ #if defined(LIBARCH_ppc64) || defined(LIBARCH_ppc64le) res = "powerpc:common64"; #endif +#ifdef LIBARCH_arm + res = "arm"; +#endif #ifdef LIBARCH_aarch64 res = "aarch64"; #endif diff -r 2e49bb347cbc -r b9a409ea423b hotspot/src/share/vm/gc/g1/g1CardLiveData.cpp --- a/hotspot/src/share/vm/gc/g1/g1CardLiveData.cpp Mon Jul 03 14:33:00 2017 +0200 +++ b/hotspot/src/share/vm/gc/g1/g1CardLiveData.cpp Fri Aug 04 12:24:33 2017 -0700 @@ -221,7 +221,7 @@ } if (hr->is_humongous()) { HeapRegion* start_region = hr->humongous_start_region(); - if (mark_bitmap->isMarked(start_region->bottom())) { + if (mark_bitmap->is_marked(start_region->bottom())) { mark_card_bitmap_range(start, hr->top()); return pointer_delta(hr->top(), start, 1); } else { @@ -236,7 +236,7 @@ p2i(start), p2i(ntams), p2i(hr->end())); // Find the first marked object at or after "start". - start = mark_bitmap->getNextMarkedWordAddress(start, ntams); + start = mark_bitmap->get_next_marked_addr(start, ntams); while (start < ntams) { oop obj = oop(start); size_t obj_size = obj->size(); @@ -250,7 +250,7 @@ marked_bytes += obj_size * HeapWordSize; // Find the next marked object after this one. - start = mark_bitmap->getNextMarkedWordAddress(obj_end, ntams); + start = mark_bitmap->get_next_marked_addr(obj_end, ntams); } return marked_bytes; diff -r 2e49bb347cbc -r b9a409ea423b hotspot/src/share/vm/gc/g1/g1CollectedHeap.cpp --- a/hotspot/src/share/vm/gc/g1/g1CollectedHeap.cpp Mon Jul 03 14:33:00 2017 +0200 +++ b/hotspot/src/share/vm/gc/g1/g1CollectedHeap.cpp Fri Aug 04 12:24:33 2017 -0700 @@ -4945,7 +4945,7 @@ p2i(r->bottom()), r->rem_set()->occupied(), r->rem_set()->strong_code_roots_list_length(), - next_bitmap->isMarked(r->bottom()), + next_bitmap->is_marked(r->bottom()), g1h->is_humongous_reclaim_candidate(region_idx), obj->is_typeArray() ); @@ -4962,13 +4962,13 @@ p2i(r->bottom()), r->rem_set()->occupied(), r->rem_set()->strong_code_roots_list_length(), - next_bitmap->isMarked(r->bottom()), + next_bitmap->is_marked(r->bottom()), g1h->is_humongous_reclaim_candidate(region_idx), obj->is_typeArray() ); // Need to clear mark bit of the humongous object if already set. - if (next_bitmap->isMarked(r->bottom())) { + if (next_bitmap->is_marked(r->bottom())) { next_bitmap->clear(r->bottom()); } _humongous_objects_reclaimed++; diff -r 2e49bb347cbc -r b9a409ea423b hotspot/src/share/vm/gc/g1/g1CollectedHeap.inline.hpp --- a/hotspot/src/share/vm/gc/g1/g1CollectedHeap.inline.hpp Mon Jul 03 14:33:00 2017 +0200 +++ b/hotspot/src/share/vm/gc/g1/g1CollectedHeap.inline.hpp Fri Aug 04 12:24:33 2017 -0700 @@ -135,7 +135,7 @@ } inline bool G1CollectedHeap::isMarkedNext(oop obj) const { - return _cm->nextMarkBitMap()->isMarked((HeapWord *)obj); + return _cm->nextMarkBitMap()->is_marked((HeapWord*)obj); } inline bool G1CollectedHeap::is_in_cset(oop obj) { diff -r 2e49bb347cbc -r b9a409ea423b hotspot/src/share/vm/gc/g1/g1ConcurrentMark.cpp --- a/hotspot/src/share/vm/gc/g1/g1ConcurrentMark.cpp Mon Jul 03 14:33:00 2017 +0200 +++ b/hotspot/src/share/vm/gc/g1/g1ConcurrentMark.cpp Fri Aug 04 12:24:33 2017 -0700 @@ -60,76 +60,21 @@ #include "utilities/align.hpp" #include "utilities/growableArray.hpp" -// Concurrent marking bit map wrapper - -G1CMBitMapRO::G1CMBitMapRO(int shifter) : - _bm(), - _shifter(shifter) { - _bmStartWord = 0; - _bmWordSize = 0; -} - -HeapWord* G1CMBitMapRO::getNextMarkedWordAddress(const HeapWord* addr, - const HeapWord* limit) const { - // First we must round addr *up* to a possible object boundary. - addr = align_up(addr, HeapWordSize << _shifter); - size_t addrOffset = heapWordToOffset(addr); - assert(limit != NULL, "limit must not be NULL"); - size_t limitOffset = heapWordToOffset(limit); - size_t nextOffset = _bm.get_next_one_offset(addrOffset, limitOffset); - HeapWord* nextAddr = offsetToHeapWord(nextOffset); - assert(nextAddr >= addr, "get_next_one postcondition"); - assert(nextAddr == limit || isMarked(nextAddr), - "get_next_one postcondition"); - return nextAddr; -} - -#ifndef PRODUCT -bool G1CMBitMapRO::covers(MemRegion heap_rs) const { - // assert(_bm.map() == _virtual_space.low(), "map inconsistency"); - assert(((size_t)_bm.size() * ((size_t)1 << _shifter)) == _bmWordSize, - "size inconsistency"); - return _bmStartWord == (HeapWord*)(heap_rs.start()) && - _bmWordSize == heap_rs.word_size(); -} -#endif - -void G1CMBitMapRO::print_on_error(outputStream* st, const char* prefix) const { - _bm.print_on_error(st, prefix); -} - -size_t G1CMBitMap::compute_size(size_t heap_size) { - return ReservedSpace::allocation_align_size_up(heap_size / mark_distance()); -} - -size_t G1CMBitMap::mark_distance() { - return MinObjAlignmentInBytes * BitsPerByte; -} - -void G1CMBitMap::initialize(MemRegion heap, G1RegionToSpaceMapper* storage) { - _bmStartWord = heap.start(); - _bmWordSize = heap.word_size(); - - _bm = BitMapView((BitMap::bm_word_t*) storage->reserved().start(), _bmWordSize >> _shifter); - - storage->set_mapping_changed_listener(&_listener); -} - -void G1CMBitMapMappingChangedListener::on_commit(uint start_region, size_t num_regions, bool zero_filled) { - if (zero_filled) { - return; - } - // We need to clear the bitmap on commit, removing any existing information. - MemRegion mr(G1CollectedHeap::heap()->bottom_addr_for_region(start_region), num_regions * HeapRegion::GrainWords); - _bm->clear_range(mr); -} - -void G1CMBitMap::clear_range(MemRegion mr) { - mr.intersection(MemRegion(_bmStartWord, _bmWordSize)); - assert(!mr.is_empty(), "unexpected empty region"); - // convert address range into offset range - _bm.at_put_range(heapWordToOffset(mr.start()), - heapWordToOffset(mr.end()), false); +bool G1CMBitMapClosure::do_addr(HeapWord* const addr) { + assert(addr < _cm->finger(), "invariant"); + assert(addr >= _task->finger(), "invariant"); + + // We move that task's local finger along. + _task->move_finger_to(addr); + + _task->scan_task_entry(G1TaskQueueEntry::from_oop(oop(addr))); + // we only partially drain the local queue and global stack + _task->drain_local_queue(true); + _task->drain_global_stack(true); + + // if the has_aborted flag has been raised, we need to bail out of + // the iteration + return !_task->has_aborted(); } G1CMMarkStack::G1CMMarkStack() : @@ -438,8 +383,6 @@ } assert(CGC_lock != NULL, "Where's the CGC_lock?"); - assert(_markBitMap1.covers(g1h->reserved_region()), "_markBitMap1 inconsistency"); - assert(_markBitMap2.covers(g1h->reserved_region()), "_markBitMap2 inconsistency"); SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set(); satb_qs.set_buffer_size(G1SATBBufferSize); @@ -753,7 +696,7 @@ void G1ConcurrentMark::clear_prev_bitmap(WorkGang* workers) { assert(SafepointSynchronize::is_at_safepoint(), "Should only clear the entire prev bitmap at a safepoint."); - clear_bitmap((G1CMBitMap*)_prevMarkBitMap, workers, false); + clear_bitmap(_prevMarkBitMap, workers, false); } class CheckBitmapClearHRClosure : public HeapRegionClosure { @@ -769,7 +712,7 @@ // value passed to it as limit to detect any found bits. // end never changes in G1. HeapWord* end = r->end(); - return _bitmap->getNextMarkedWordAddress(r->bottom(), end) != end; + return _bitmap->get_next_marked_addr(r->bottom(), end) != end; } }; @@ -789,7 +732,6 @@ void G1ConcurrentMark::checkpointRootsInitialPre() { G1CollectedHeap* g1h = G1CollectedHeap::heap(); - G1Policy* g1p = g1h->g1_policy(); _has_aborted = false; @@ -1764,9 +1706,9 @@ } void G1ConcurrentMark::swapMarkBitMaps() { - G1CMBitMapRO* temp = _prevMarkBitMap; - _prevMarkBitMap = (G1CMBitMapRO*)_nextMarkBitMap; - _nextMarkBitMap = (G1CMBitMap*) temp; + G1CMBitMap* temp = _prevMarkBitMap; + _prevMarkBitMap = _nextMarkBitMap; + _nextMarkBitMap = temp; } // Closure for marking entries in SATB buffers. @@ -1780,15 +1722,8 @@ // circumspect about treating the argument as an object. void do_entry(void* entry) const { _task->increment_refs_reached(); - HeapRegion* hr = _g1h->heap_region_containing(entry); - if (entry < hr->next_top_at_mark_start()) { - // Until we get here, we don't know whether entry refers to a valid - // object; it could instead have been a stale reference. - oop obj = static_cast(entry); - assert(obj->is_oop(true /* ignore mark word */), - "Invalid oop in SATB buffer: " PTR_FORMAT, p2i(obj)); - _task->make_reference_grey(obj); - } + oop const obj = static_cast(entry); + _task->make_reference_grey(obj); } public: @@ -1911,9 +1846,7 @@ } void G1ConcurrentMark::clearRangePrevBitmap(MemRegion mr) { - // Note we are overriding the read-only view of the prev map here, via - // the cast. - ((G1CMBitMap*)_prevMarkBitMap)->clear_range(mr); + _prevMarkBitMap->clear_range(mr); } HeapRegion* @@ -2160,38 +2093,6 @@ _nextMarkBitMap->print_on_error(st, " Next Bits: "); } -// Closure for iteration over bitmaps -class G1CMBitMapClosure : public BitMapClosure { -private: - // the bitmap that is being iterated over - G1CMBitMap* _nextMarkBitMap; - G1ConcurrentMark* _cm; - G1CMTask* _task; - -public: - G1CMBitMapClosure(G1CMTask *task, G1ConcurrentMark* cm, G1CMBitMap* nextMarkBitMap) : - _task(task), _cm(cm), _nextMarkBitMap(nextMarkBitMap) { } - - bool do_bit(size_t offset) { - HeapWord* addr = _nextMarkBitMap->offsetToHeapWord(offset); - assert(_nextMarkBitMap->isMarked(addr), "invariant"); - assert( addr < _cm->finger(), "invariant"); - assert(addr >= _task->finger(), "invariant"); - - // We move that task's local finger along. - _task->move_finger_to(addr); - - _task->scan_task_entry(G1TaskQueueEntry::from_oop(oop(addr))); - // we only partially drain the local queue and global stack - _task->drain_local_queue(true); - _task->drain_global_stack(true); - - // if the has_aborted flag has been raised, we need to bail out of - // the iteration - return !_task->has_aborted(); - } -}; - static ReferenceProcessor* get_cm_oop_closure_ref_processor(G1CollectedHeap* g1h) { ReferenceProcessor* result = g1h->ref_processor_cm(); assert(result != NULL, "CM reference processor should not be NULL"); @@ -2691,7 +2592,7 @@ // Set up the bitmap and oop closures. Anything that uses them is // eventually called from this method, so it is OK to allocate these // statically. - G1CMBitMapClosure bitmap_closure(this, _cm, _nextMarkBitMap); + G1CMBitMapClosure bitmap_closure(this, _cm); G1CMOopClosure cm_oop_closure(_g1h, _cm, this); set_cm_oop_closure(&cm_oop_closure); @@ -2747,10 +2648,9 @@ giveup_current_region(); regular_clock_call(); } else if (_curr_region->is_humongous() && mr.start() == _curr_region->bottom()) { - if (_nextMarkBitMap->isMarked(mr.start())) { + if (_nextMarkBitMap->is_marked(mr.start())) { // The object is marked - apply the closure - BitMap::idx_t offset = _nextMarkBitMap->heapWordToOffset(mr.start()); - bitmap_closure.do_bit(offset); + bitmap_closure.do_addr(mr.start()); } // Even if this task aborted while scanning the humongous object // we can (and should) give up the current region. @@ -2772,11 +2672,9 @@ // points to the address of the object we last scanned. If we // leave it there, when we restart this task, we will rescan // the object. It is easy to avoid this. We move the finger by - // enough to point to the next possible object header (the - // bitmap knows by how much we need to move it as it knows its - // granularity). + // enough to point to the next possible object header. assert(_finger < _region_limit, "invariant"); - HeapWord* new_finger = _nextMarkBitMap->nextObject(_finger); + HeapWord* const new_finger = _finger + ((oop)_finger)->size(); // Check if bitmap iteration was aborted while scanning the last object if (new_finger >= _region_limit) { giveup_current_region(); diff -r 2e49bb347cbc -r b9a409ea423b hotspot/src/share/vm/gc/g1/g1ConcurrentMark.hpp --- a/hotspot/src/share/vm/gc/g1/g1ConcurrentMark.hpp Mon Jul 03 14:33:00 2017 +0200 +++ b/hotspot/src/share/vm/gc/g1/g1ConcurrentMark.hpp Fri Aug 04 12:24:33 2017 -0700 @@ -26,13 +26,13 @@ #define SHARE_VM_GC_G1_G1CONCURRENTMARK_HPP #include "classfile/javaClasses.hpp" +#include "gc/g1/g1ConcurrentMarkBitMap.hpp" #include "gc/g1/g1ConcurrentMarkObjArrayProcessor.hpp" #include "gc/g1/g1RegionToSpaceMapper.hpp" #include "gc/g1/heapRegionSet.hpp" #include "gc/shared/taskqueue.hpp" class G1CollectedHeap; -class G1CMBitMap; class G1CMTask; class G1ConcurrentMark; class ConcurrentGCTimer; @@ -110,97 +110,6 @@ bool do_object_b(oop obj); }; -// A generic CM bit map. This is essentially a wrapper around the BitMap -// class, with one bit per (1<<_shifter) HeapWords. - -class G1CMBitMapRO VALUE_OBJ_CLASS_SPEC { - protected: - HeapWord* _bmStartWord; // base address of range covered by map - size_t _bmWordSize; // map size (in #HeapWords covered) - const int _shifter; // map to char or bit - BitMapView _bm; // the bit map itself - - public: - // constructor - G1CMBitMapRO(int shifter); - - // inquiries - HeapWord* startWord() const { return _bmStartWord; } - // the following is one past the last word in space - HeapWord* endWord() const { return _bmStartWord + _bmWordSize; } - - // read marks - - bool isMarked(HeapWord* addr) const { - assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), - "outside underlying space?"); - return _bm.at(heapWordToOffset(addr)); - } - - // iteration - inline bool iterate(BitMapClosure* cl, MemRegion mr); - - // Return the address corresponding to the next marked bit at or after - // "addr", and before "limit", if "limit" is non-NULL. If there is no - // such bit, returns "limit" if that is non-NULL, or else "endWord()". - HeapWord* getNextMarkedWordAddress(const HeapWord* addr, - const HeapWord* limit = NULL) const; - - // conversion utilities - HeapWord* offsetToHeapWord(size_t offset) const { - return _bmStartWord + (offset << _shifter); - } - size_t heapWordToOffset(const HeapWord* addr) const { - return pointer_delta(addr, _bmStartWord) >> _shifter; - } - - // The argument addr should be the start address of a valid object - inline HeapWord* nextObject(HeapWord* addr); - - void print_on_error(outputStream* st, const char* prefix) const; - - // debugging - NOT_PRODUCT(bool covers(MemRegion rs) const;) -}; - -class G1CMBitMapMappingChangedListener : public G1MappingChangedListener { - private: - G1CMBitMap* _bm; - public: - G1CMBitMapMappingChangedListener() : _bm(NULL) {} - - void set_bitmap(G1CMBitMap* bm) { _bm = bm; } - - virtual void on_commit(uint start_idx, size_t num_regions, bool zero_filled); -}; - -class G1CMBitMap : public G1CMBitMapRO { - private: - G1CMBitMapMappingChangedListener _listener; - - public: - static size_t compute_size(size_t heap_size); - // Returns the amount of bytes on the heap between two marks in the bitmap. - static size_t mark_distance(); - // Returns how many bytes (or bits) of the heap a single byte (or bit) of the - // mark bitmap corresponds to. This is the same as the mark distance above. - static size_t heap_map_factor() { - return mark_distance(); - } - - G1CMBitMap() : G1CMBitMapRO(LogMinObjAlignment), _listener() { _listener.set_bitmap(this); } - - // Initializes the underlying BitMap to cover the given area. - void initialize(MemRegion heap, G1RegionToSpaceMapper* storage); - - // Write marks. - inline void mark(HeapWord* addr); - inline void clear(HeapWord* addr); - inline bool parMark(HeapWord* addr); - - void clear_range(MemRegion mr); -}; - // Represents the overflow mark stack used by concurrent marking. // // Stores oops in a huge buffer in virtual memory that is always fully committed. @@ -396,7 +305,7 @@ // Concurrent marking support structures G1CMBitMap _markBitMap1; G1CMBitMap _markBitMap2; - G1CMBitMapRO* _prevMarkBitMap; // Completed mark bitmap + G1CMBitMap* _prevMarkBitMap; // Completed mark bitmap G1CMBitMap* _nextMarkBitMap; // Under-construction mark bitmap // Heap bounds @@ -626,8 +535,8 @@ ConcurrentMarkThread* cmThread() { return _cmThread; } - G1CMBitMapRO* prevMarkBitMap() const { return _prevMarkBitMap; } - G1CMBitMap* nextMarkBitMap() const { return _nextMarkBitMap; } + const G1CMBitMap* const prevMarkBitMap() const { return _prevMarkBitMap; } + G1CMBitMap* nextMarkBitMap() const { return _nextMarkBitMap; } // Returns the number of GC threads to be used in a concurrent // phase based on the number of GC threads being used in a STW @@ -637,16 +546,6 @@ // Calculates the number of GC threads to be used in a concurrent phase. uint calc_parallel_marking_threads(); - // The following three are interaction between CM and - // G1CollectedHeap - - // This notifies CM that a root during initial-mark needs to be - // grayed. It is MT-safe. hr is the region that - // contains the object and it's passed optionally from callers who - // might already have it (no point in recalculating it). - inline void grayRoot(oop obj, - HeapRegion* hr = NULL); - // Prepare internal data structures for the next mark cycle. This includes clearing // the next mark bitmap and some internal data structures. This method is intended // to be called concurrently to the mutator. It will yield to safepoint requests. @@ -713,8 +612,9 @@ void print_on_error(outputStream* st) const; - // Attempts to mark the given object on the next mark bitmap. - inline bool par_mark(oop obj); + // Mark the given object on the next bitmap if it is below nTAMS. + inline bool mark_in_next_bitmap(HeapRegion* const hr, oop const obj); + inline bool mark_in_next_bitmap(oop const obj); // Returns true if initialization was successfully completed. bool completed_initialization() const { diff -r 2e49bb347cbc -r b9a409ea423b hotspot/src/share/vm/gc/g1/g1ConcurrentMark.inline.hpp --- a/hotspot/src/share/vm/gc/g1/g1ConcurrentMark.inline.hpp Mon Jul 03 14:33:00 2017 +0200 +++ b/hotspot/src/share/vm/gc/g1/g1ConcurrentMark.inline.hpp Fri Aug 04 12:24:33 2017 -0700 @@ -27,70 +27,38 @@ #include "gc/g1/g1CollectedHeap.inline.hpp" #include "gc/g1/g1ConcurrentMark.hpp" +#include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp" #include "gc/g1/g1ConcurrentMarkObjArrayProcessor.inline.hpp" #include "gc/g1/suspendibleThreadSet.hpp" #include "gc/shared/taskqueue.inline.hpp" #include "utilities/bitMap.inline.hpp" -inline bool G1ConcurrentMark::par_mark(oop obj) { - return _nextMarkBitMap->parMark((HeapWord*)obj); -} - -inline bool G1CMBitMapRO::iterate(BitMapClosure* cl, MemRegion mr) { - HeapWord* start_addr = MAX2(startWord(), mr.start()); - HeapWord* end_addr = MIN2(endWord(), mr.end()); - - if (end_addr > start_addr) { - // Right-open interval [start-offset, end-offset). - BitMap::idx_t start_offset = heapWordToOffset(start_addr); - BitMap::idx_t end_offset = heapWordToOffset(end_addr); - - start_offset = _bm.get_next_one_offset(start_offset, end_offset); - while (start_offset < end_offset) { - if (!cl->do_bit(start_offset)) { - return false; - } - HeapWord* next_addr = MIN2(nextObject(offsetToHeapWord(start_offset)), end_addr); - BitMap::idx_t next_offset = heapWordToOffset(next_addr); - start_offset = _bm.get_next_one_offset(next_offset, end_offset); - } - } - return true; +inline bool G1ConcurrentMark::mark_in_next_bitmap(oop const obj) { + HeapRegion* const hr = _g1h->heap_region_containing(obj); + return mark_in_next_bitmap(hr, obj); } -// The argument addr should be the start address of a valid object -HeapWord* G1CMBitMapRO::nextObject(HeapWord* addr) { - oop obj = (oop) addr; - HeapWord* res = addr + obj->size(); - assert(offsetToHeapWord(heapWordToOffset(res)) == res, "sanity"); - return res; -} +inline bool G1ConcurrentMark::mark_in_next_bitmap(HeapRegion* const hr, oop const obj) { + assert(hr != NULL, "just checking"); + assert(hr->is_in_reserved(obj), "Attempting to mark object at " PTR_FORMAT " that is not contained in the given region %u", p2i(obj), hr->hrm_index()); -#define check_mark(addr) \ - assert(_bmStartWord <= (addr) && (addr) < (_bmStartWord + _bmWordSize), \ - "outside underlying space?"); \ - assert(G1CollectedHeap::heap()->is_in_exact(addr), \ - "Trying to access not available bitmap " PTR_FORMAT \ - " corresponding to " PTR_FORMAT " (%u)", \ - p2i(this), p2i(addr), G1CollectedHeap::heap()->addr_to_region(addr)); + if (hr->obj_allocated_since_next_marking(obj)) { + return false; + } -inline void G1CMBitMap::mark(HeapWord* addr) { - check_mark(addr); - _bm.set_bit(heapWordToOffset(addr)); -} + // Some callers may have stale objects to mark above nTAMS after humongous reclaim. + assert(obj->is_oop(true /* ignore mark word */), "Address " PTR_FORMAT " to mark is not an oop", p2i(obj)); + assert(!hr->is_continues_humongous(), "Should not try to mark object " PTR_FORMAT " in Humongous continues region %u above nTAMS " PTR_FORMAT, p2i(obj), hr->hrm_index(), p2i(hr->next_top_at_mark_start())); -inline void G1CMBitMap::clear(HeapWord* addr) { - check_mark(addr); - _bm.clear_bit(heapWordToOffset(addr)); -} + HeapWord* const obj_addr = (HeapWord*)obj; + // Dirty read to avoid CAS. + if (_nextMarkBitMap->is_marked(obj_addr)) { + return false; + } -inline bool G1CMBitMap::parMark(HeapWord* addr) { - check_mark(addr); - return _bm.par_set_bit(heapWordToOffset(addr)); + return _nextMarkBitMap->par_mark(obj_addr); } -#undef check_mark - #ifndef PRODUCT template inline void G1CMMarkStack::iterate(Fn fn) const { @@ -122,7 +90,7 @@ assert(task_entry.is_array_slice() || !_g1h->is_on_master_free_list( _g1h->heap_region_containing(task_entry.obj())), "invariant"); assert(task_entry.is_array_slice() || !_g1h->is_obj_ill(task_entry.obj()), "invariant"); // FIXME!!! - assert(task_entry.is_array_slice() || _nextMarkBitMap->isMarked((HeapWord*)task_entry.obj()), "invariant"); + assert(task_entry.is_array_slice() || _nextMarkBitMap->is_marked((HeapWord*)task_entry.obj()), "invariant"); if (!_task_queue->push(task_entry)) { // The local task queue looks full. We need to push some entries @@ -170,7 +138,7 @@ template inline void G1CMTask::process_grey_task_entry(G1TaskQueueEntry task_entry) { assert(scan || (task_entry.is_oop() && task_entry.obj()->is_typeArray()), "Skipping scan of grey non-typeArray"); - assert(task_entry.is_array_slice() || _nextMarkBitMap->isMarked((HeapWord*)task_entry.obj()), + assert(task_entry.is_array_slice() || _nextMarkBitMap->is_marked((HeapWord*)task_entry.obj()), "Any stolen object should be a slice or marked"); if (scan) { @@ -194,98 +162,63 @@ } inline void G1CMTask::make_reference_grey(oop obj) { - if (_cm->par_mark(obj)) { - // No OrderAccess:store_load() is needed. It is implicit in the - // CAS done in G1CMBitMap::parMark() call in the routine above. - HeapWord* global_finger = _cm->finger(); + if (!_cm->mark_in_next_bitmap(obj)) { + return; + } + + // No OrderAccess:store_load() is needed. It is implicit in the + // CAS done in G1CMBitMap::parMark() call in the routine above. + HeapWord* global_finger = _cm->finger(); - // We only need to push a newly grey object on the mark - // stack if it is in a section of memory the mark bitmap - // scan has already examined. Mark bitmap scanning - // maintains progress "fingers" for determining that. - // - // Notice that the global finger might be moving forward - // concurrently. This is not a problem. In the worst case, we - // mark the object while it is above the global finger and, by - // the time we read the global finger, it has moved forward - // past this object. In this case, the object will probably - // be visited when a task is scanning the region and will also - // be pushed on the stack. So, some duplicate work, but no - // correctness problems. - if (is_below_finger(obj, global_finger)) { - G1TaskQueueEntry entry = G1TaskQueueEntry::from_oop(obj); - if (obj->is_typeArray()) { - // Immediately process arrays of primitive types, rather - // than pushing on the mark stack. This keeps us from - // adding humongous objects to the mark stack that might - // be reclaimed before the entry is processed - see - // selection of candidates for eager reclaim of humongous - // objects. The cost of the additional type test is - // mitigated by avoiding a trip through the mark stack, - // by only doing a bookkeeping update and avoiding the - // actual scan of the object - a typeArray contains no - // references, and the metadata is built-in. - process_grey_task_entry(entry); - } else { - push(entry); - } + // We only need to push a newly grey object on the mark + // stack if it is in a section of memory the mark bitmap + // scan has already examined. Mark bitmap scanning + // maintains progress "fingers" for determining that. + // + // Notice that the global finger might be moving forward + // concurrently. This is not a problem. In the worst case, we + // mark the object while it is above the global finger and, by + // the time we read the global finger, it has moved forward + // past this object. In this case, the object will probably + // be visited when a task is scanning the region and will also + // be pushed on the stack. So, some duplicate work, but no + // correctness problems. + if (is_below_finger(obj, global_finger)) { + G1TaskQueueEntry entry = G1TaskQueueEntry::from_oop(obj); + if (obj->is_typeArray()) { + // Immediately process arrays of primitive types, rather + // than pushing on the mark stack. This keeps us from + // adding humongous objects to the mark stack that might + // be reclaimed before the entry is processed - see + // selection of candidates for eager reclaim of humongous + // objects. The cost of the additional type test is + // mitigated by avoiding a trip through the mark stack, + // by only doing a bookkeeping update and avoiding the + // actual scan of the object - a typeArray contains no + // references, and the metadata is built-in. + process_grey_task_entry(entry); + } else { + push(entry); } } } inline void G1CMTask::deal_with_reference(oop obj) { increment_refs_reached(); - - HeapWord* objAddr = (HeapWord*) obj; - assert(obj->is_oop_or_null(true /* ignore mark word */), "Expected an oop or NULL at " PTR_FORMAT, p2i(obj)); - if (_g1h->is_in_g1_reserved(objAddr)) { - assert(obj != NULL, "null check is implicit"); - if (!_nextMarkBitMap->isMarked(objAddr)) { - // Only get the containing region if the object is not marked on the - // bitmap (otherwise, it's a waste of time since we won't do - // anything with it). - HeapRegion* hr = _g1h->heap_region_containing(obj); - if (!hr->obj_allocated_since_next_marking(obj)) { - make_reference_grey(obj); - } - } + if (obj == NULL) { + return; } + make_reference_grey(obj); } inline void G1ConcurrentMark::markPrev(oop p) { - assert(!_prevMarkBitMap->isMarked((HeapWord*) p), "sanity"); - // Note we are overriding the read-only view of the prev map here, via - // the cast. - ((G1CMBitMap*)_prevMarkBitMap)->mark((HeapWord*) p); + assert(!_prevMarkBitMap->is_marked((HeapWord*) p), "sanity"); + _prevMarkBitMap->mark((HeapWord*) p); } bool G1ConcurrentMark::isPrevMarked(oop p) const { assert(p != NULL && p->is_oop(), "expected an oop"); - HeapWord* addr = (HeapWord*)p; - assert(addr >= _prevMarkBitMap->startWord() || - addr < _prevMarkBitMap->endWord(), "in a region"); - - return _prevMarkBitMap->isMarked(addr); -} - -inline void G1ConcurrentMark::grayRoot(oop obj, HeapRegion* hr) { - assert(obj != NULL, "pre-condition"); - HeapWord* addr = (HeapWord*) obj; - if (hr == NULL) { - hr = _g1h->heap_region_containing(addr); - } else { - assert(hr->is_in(addr), "pre-condition"); - } - assert(hr != NULL, "sanity"); - // Given that we're looking for a region that contains an object - // header it's impossible to get back a HC region. - assert(!hr->is_continues_humongous(), "sanity"); - - if (addr < hr->next_top_at_mark_start()) { - if (!_nextMarkBitMap->isMarked(addr)) { - par_mark(obj); - } - } + return _prevMarkBitMap->is_marked((HeapWord*)p); } inline bool G1ConcurrentMark::do_yield_check() { diff -r 2e49bb347cbc -r b9a409ea423b hotspot/src/share/vm/gc/g1/g1ConcurrentMarkBitMap.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/hotspot/src/share/vm/gc/g1/g1ConcurrentMarkBitMap.cpp Fri Aug 04 12:24:33 2017 -0700 @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp" +#include "memory/virtualspace.hpp" + +void G1CMBitMap::print_on_error(outputStream* st, const char* prefix) const { + _bm.print_on_error(st, prefix); +} + +size_t G1CMBitMap::compute_size(size_t heap_size) { + return ReservedSpace::allocation_align_size_up(heap_size / mark_distance()); +} + +size_t G1CMBitMap::mark_distance() { + return MinObjAlignmentInBytes * BitsPerByte; +} + +void G1CMBitMap::initialize(MemRegion heap, G1RegionToSpaceMapper* storage) { + _covered = heap; + + _bm = BitMapView((BitMap::bm_word_t*) storage->reserved().start(), _covered.word_size() >> _shifter); + + storage->set_mapping_changed_listener(&_listener); +} + +void G1CMBitMapMappingChangedListener::on_commit(uint start_region, size_t num_regions, bool zero_filled) { + if (zero_filled) { + return; + } + // We need to clear the bitmap on commit, removing any existing information. + MemRegion mr(G1CollectedHeap::heap()->bottom_addr_for_region(start_region), num_regions * HeapRegion::GrainWords); + _bm->clear_range(mr); +} + +void G1CMBitMap::clear_range(MemRegion mr) { + MemRegion intersection = mr.intersection(_covered); + assert(!intersection.is_empty(), + "Given range from " PTR_FORMAT " to " PTR_FORMAT " is completely outside the heap", + p2i(mr.start()), p2i(mr.end())); + // convert address range into offset range + _bm.at_put_range(addr_to_offset(intersection.start()), + addr_to_offset(intersection.end()), false); +} diff -r 2e49bb347cbc -r b9a409ea423b hotspot/src/share/vm/gc/g1/g1ConcurrentMarkBitMap.hpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/hotspot/src/share/vm/gc/g1/g1ConcurrentMarkBitMap.hpp Fri Aug 04 12:24:33 2017 -0700 @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1CONCURRENTMARKBITMAP_HPP +#define SHARE_VM_GC_G1_G1CONCURRENTMARKBITMAP_HPP + +#include "gc/g1/g1RegionToSpaceMapper.hpp" +#include "memory/allocation.hpp" +#include "memory/memRegion.hpp" +#include "utilities/bitMap.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/macros.hpp" + +class G1CMBitMap; +class G1CMTask; +class G1ConcurrentMark; + +// Closure for iteration over bitmaps +class G1CMBitMapClosure VALUE_OBJ_CLASS_SPEC { +private: + G1ConcurrentMark* const _cm; + G1CMTask* const _task; +public: + G1CMBitMapClosure(G1CMTask *task, G1ConcurrentMark* cm) : _task(task), _cm(cm) { } + + bool do_addr(HeapWord* const addr); +}; + +class G1CMBitMapMappingChangedListener : public G1MappingChangedListener { + private: + G1CMBitMap* _bm; + public: + G1CMBitMapMappingChangedListener() : _bm(NULL) {} + + void set_bitmap(G1CMBitMap* bm) { _bm = bm; } + + virtual void on_commit(uint start_idx, size_t num_regions, bool zero_filled); +}; + +// A generic mark bitmap for concurrent marking. This is essentially a wrapper +// around the BitMap class that is based on HeapWords, with one bit per (1 << _shifter) HeapWords. +class G1CMBitMap VALUE_OBJ_CLASS_SPEC { +private: + MemRegion _covered; // The heap area covered by this bitmap. + + const int _shifter; // Shift amount from heap index to bit index in the bitmap. + + BitMapView _bm; // The actual bitmap. + + G1CMBitMapMappingChangedListener _listener; + + inline void check_mark(HeapWord* addr) NOT_DEBUG_RETURN; + + // Convert from bit offset to address. + HeapWord* offset_to_addr(size_t offset) const { + return _covered.start() + (offset << _shifter); + } + // Convert from address to bit offset. + size_t addr_to_offset(const HeapWord* addr) const { + return pointer_delta(addr, _covered.start()) >> _shifter; + } +public: + static size_t compute_size(size_t heap_size); + // Returns the amount of bytes on the heap between two marks in the bitmap. + static size_t mark_distance(); + // Returns how many bytes (or bits) of the heap a single byte (or bit) of the + // mark bitmap corresponds to. This is the same as the mark distance above. + static size_t heap_map_factor() { + return mark_distance(); + } + + G1CMBitMap() : _covered(), _bm(), _shifter(LogMinObjAlignment), _listener() { _listener.set_bitmap(this); } + + // Initializes the underlying BitMap to cover the given area. + void initialize(MemRegion heap, G1RegionToSpaceMapper* storage); + + // Read marks + bool is_marked(HeapWord* addr) const { + assert(_covered.contains(addr), + "Address " PTR_FORMAT " is outside underlying space from " PTR_FORMAT " to " PTR_FORMAT, + p2i(addr), p2i(_covered.start()), p2i(_covered.end())); + return _bm.at(addr_to_offset(addr)); + } + + // Apply the closure to the addresses that correspond to marked bits in the bitmap. + inline bool iterate(G1CMBitMapClosure* cl, MemRegion mr); + + // Return the address corresponding to the next marked bit at or after + // "addr", and before "limit", if "limit" is non-NULL. If there is no + // such bit, returns "limit" if that is non-NULL, or else "endWord()". + inline HeapWord* get_next_marked_addr(const HeapWord* addr, + const HeapWord* limit) const; + + // The argument addr should be the start address of a valid object + inline HeapWord* addr_after_obj(HeapWord* addr); + + void print_on_error(outputStream* st, const char* prefix) const; + + // Write marks. + inline void mark(HeapWord* addr); + inline void clear(HeapWord* addr); + inline bool par_mark(HeapWord* addr); + + void clear_range(MemRegion mr); +}; + +#endif // SHARE_VM_GC_G1_G1CONCURRENTMARKBITMAP_HPP diff -r 2e49bb347cbc -r b9a409ea423b hotspot/src/share/vm/gc/g1/g1ConcurrentMarkBitMap.inline.hpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/hotspot/src/share/vm/gc/g1/g1ConcurrentMarkBitMap.inline.hpp Fri Aug 04 12:24:33 2017 -0700 @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1CONCURRENTMARKBITMAP_INLINE_HPP +#define SHARE_VM_GC_G1_G1CONCURRENTMARKBITMAP_INLINE_HPP + +#include "gc/g1/g1ConcurrentMarkBitMap.hpp" +#include "memory/memRegion.hpp" +#include "utilities/align.hpp" +#include "utilities/bitMap.inline.hpp" + +inline bool G1CMBitMap::iterate(G1CMBitMapClosure* cl, MemRegion mr) { + assert(!mr.is_empty(), "Does not support empty memregion to iterate over"); + assert(_covered.contains(mr), + "Given MemRegion from " PTR_FORMAT " to " PTR_FORMAT " not contained in heap area", + p2i(mr.start()), p2i(mr.end())); + + BitMap::idx_t const end_offset = addr_to_offset(mr.end()); + BitMap::idx_t offset = _bm.get_next_one_offset(addr_to_offset(mr.start()), end_offset); + + while (offset < end_offset) { + HeapWord* const addr = offset_to_addr(offset); + if (!cl->do_addr(addr)) { + return false; + } + size_t const obj_size = (size_t)((oop)addr)->size(); + offset = _bm.get_next_one_offset(offset + (obj_size >> _shifter), end_offset); + } + return true; +} + +inline HeapWord* G1CMBitMap::get_next_marked_addr(const HeapWord* addr, + const HeapWord* limit) const { + assert(limit != NULL, "limit must not be NULL"); + // Round addr up to a possible object boundary to be safe. + size_t const addr_offset = addr_to_offset(align_up(addr, HeapWordSize << _shifter)); + size_t const limit_offset = addr_to_offset(limit); + size_t const nextOffset = _bm.get_next_one_offset(addr_offset, limit_offset); + return offset_to_addr(nextOffset); +} + +#ifdef ASSERT +inline void G1CMBitMap::check_mark(HeapWord* addr) { + assert(G1CollectedHeap::heap()->is_in_exact(addr), + "Trying to access bitmap " PTR_FORMAT " for address " PTR_FORMAT " not in the heap.", + p2i(this), p2i(addr)); +} +#endif + +inline void G1CMBitMap::mark(HeapWord* addr) { + check_mark(addr); + _bm.set_bit(addr_to_offset(addr)); +} + +inline void G1CMBitMap::clear(HeapWord* addr) { + check_mark(addr); + _bm.clear_bit(addr_to_offset(addr)); +} + +inline bool G1CMBitMap::par_mark(HeapWord* addr) { + check_mark(addr); + return _bm.par_set_bit(addr_to_offset(addr)); +} + +#endif // SHARE_VM_GC_G1_G1CONCURRENTMARKBITMAP_INLINE_HPP diff -r 2e49bb347cbc -r b9a409ea423b hotspot/src/share/vm/gc/g1/g1EvacFailure.cpp --- a/hotspot/src/share/vm/gc/g1/g1EvacFailure.cpp Mon Jul 03 14:33:00 2017 +0200 +++ b/hotspot/src/share/vm/gc/g1/g1EvacFailure.cpp Fri Aug 04 12:24:33 2017 -0700 @@ -124,7 +124,7 @@ // explicitly and all objects in the CSet are considered // (implicitly) live. So, we won't mark them explicitly and // we'll leave them over NTAMS. - _cm->grayRoot(obj, _hr); + _cm->mark_in_next_bitmap(_hr, obj); } size_t obj_size = obj->size(); diff -r 2e49bb347cbc -r b9a409ea423b hotspot/src/share/vm/gc/g1/g1HeapVerifier.cpp --- a/hotspot/src/share/vm/gc/g1/g1HeapVerifier.cpp Mon Jul 03 14:33:00 2017 +0200 +++ b/hotspot/src/share/vm/gc/g1/g1HeapVerifier.cpp Fri Aug 04 12:24:33 2017 -0700 @@ -598,11 +598,11 @@ _g1h->collection_set()->iterate(&cl); } -bool G1HeapVerifier::verify_no_bits_over_tams(const char* bitmap_name, G1CMBitMapRO* bitmap, +bool G1HeapVerifier::verify_no_bits_over_tams(const char* bitmap_name, const G1CMBitMap* const bitmap, HeapWord* tams, HeapWord* end) { guarantee(tams <= end, "tams: " PTR_FORMAT " end: " PTR_FORMAT, p2i(tams), p2i(end)); - HeapWord* result = bitmap->getNextMarkedWordAddress(tams, end); + HeapWord* result = bitmap->get_next_marked_addr(tams, end); if (result < end) { log_error(gc, verify)("## wrong marked address on %s bitmap: " PTR_FORMAT, bitmap_name, p2i(result)); log_error(gc, verify)("## %s tams: " PTR_FORMAT " end: " PTR_FORMAT, bitmap_name, p2i(tams), p2i(end)); @@ -612,10 +612,9 @@ } bool G1HeapVerifier::verify_bitmaps(const char* caller, HeapRegion* hr) { - G1CMBitMapRO* prev_bitmap = _g1h->concurrent_mark()->prevMarkBitMap(); - G1CMBitMapRO* next_bitmap = (G1CMBitMapRO*) _g1h->concurrent_mark()->nextMarkBitMap(); + const G1CMBitMap* const prev_bitmap = _g1h->concurrent_mark()->prevMarkBitMap(); + const G1CMBitMap* const next_bitmap = _g1h->concurrent_mark()->nextMarkBitMap(); - HeapWord* bottom = hr->bottom(); HeapWord* ptams = hr->prev_top_at_mark_start(); HeapWord* ntams = hr->next_top_at_mark_start(); HeapWord* end = hr->end(); diff -r 2e49bb347cbc -r b9a409ea423b hotspot/src/share/vm/gc/g1/g1HeapVerifier.hpp --- a/hotspot/src/share/vm/gc/g1/g1HeapVerifier.hpp Mon Jul 03 14:33:00 2017 +0200 +++ b/hotspot/src/share/vm/gc/g1/g1HeapVerifier.hpp Fri Aug 04 12:24:33 2017 -0700 @@ -82,7 +82,7 @@ // range [from,limit). If it does, print an error message and return // false. Otherwise, just return true. bitmap_name should be "prev" // or "next". - bool verify_no_bits_over_tams(const char* bitmap_name, G1CMBitMapRO* bitmap, + bool verify_no_bits_over_tams(const char* bitmap_name, const G1CMBitMap* const bitmap, HeapWord* from, HeapWord* limit); // Verify that the prev / next bitmap range [tams,end) for the given diff -r 2e49bb347cbc -r b9a409ea423b hotspot/src/share/vm/gc/g1/g1OopClosures.inline.hpp --- a/hotspot/src/share/vm/gc/g1/g1OopClosures.inline.hpp Mon Jul 03 14:33:00 2017 +0200 +++ b/hotspot/src/share/vm/gc/g1/g1OopClosures.inline.hpp Fri Aug 04 12:24:33 2017 -0700 @@ -95,11 +95,11 @@ template inline void G1RootRegionScanClosure::do_oop_nv(T* p) { T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - HeapRegion* hr = _g1h->heap_region_containing((HeapWord*) obj); - _cm->grayRoot(obj, hr); + if (oopDesc::is_null(heap_oop)) { + return; } + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + _cm->mark_in_next_bitmap(obj); } template @@ -205,8 +205,7 @@ void G1ParCopyHelper::mark_object(oop obj) { assert(!_g1->heap_region_containing(obj)->in_collection_set(), "should not mark objects in the CSet"); - // We know that the object is not moving so it's safe to read its size. - _cm->grayRoot(obj); + _cm->mark_in_next_bitmap(obj); } void G1ParCopyHelper::mark_forwarded_object(oop from_obj, oop to_obj) { @@ -217,11 +216,7 @@ assert(_g1->heap_region_containing(from_obj)->in_collection_set(), "from obj should be in the CSet"); assert(!_g1->heap_region_containing(to_obj)->in_collection_set(), "should not mark objects in the CSet"); - // The object might be in the process of being copied by another - // worker so we cannot trust that its to-space image is - // well-formed. So we have to read its size from its from-space - // image which we know should not be changing. - _cm->grayRoot(to_obj); + _cm->mark_in_next_bitmap(to_obj); } template diff -r 2e49bb347cbc -r b9a409ea423b hotspot/src/share/vm/gc/g1/heapRegion.hpp --- a/hotspot/src/share/vm/gc/g1/heapRegion.hpp Mon Jul 03 14:33:00 2017 +0200 +++ b/hotspot/src/share/vm/gc/g1/heapRegion.hpp Fri Aug 04 12:24:33 2017 -0700 @@ -56,7 +56,7 @@ // room for filler objects to pad out to the end of the region. class G1CollectedHeap; -class G1CMBitMapRO; +class G1CMBitMap; class HeapRegionRemSet; class HeapRegionRemSetIterator; class HeapRegion; @@ -225,7 +225,7 @@ // May // - only called with obj < top() // - not called on humongous objects or archive regions - inline bool is_obj_dead_with_size(const oop obj, G1CMBitMapRO* prev_bitmap, size_t* size) const; + inline bool is_obj_dead_with_size(const oop obj, const G1CMBitMap* const prev_bitmap, size_t* size) const; protected: // The index of this region in the heap region sequence. @@ -301,7 +301,7 @@ // Returns the block size of the given (dead, potentially having its class unloaded) object // starting at p extending to at most the prev TAMS using the given mark bitmap. - inline size_t block_size_using_bitmap(const HeapWord* p, const G1CMBitMapRO* prev_bitmap) const; + inline size_t block_size_using_bitmap(const HeapWord* p, const G1CMBitMap* const prev_bitmap) const; public: HeapRegion(uint hrm_index, G1BlockOffsetTable* bot, @@ -349,7 +349,7 @@ bool block_is_obj(const HeapWord* p) const; // Returns whether the given object is dead based on TAMS and bitmap. - bool is_obj_dead(const oop obj, const G1CMBitMapRO* prev_bitmap) const; + bool is_obj_dead(const oop obj, const G1CMBitMap* const prev_bitmap) const; // Returns the object size for all valid block starts // and the amount of unallocated words if called on top() diff -r 2e49bb347cbc -r b9a409ea423b hotspot/src/share/vm/gc/g1/heapRegion.inline.hpp --- a/hotspot/src/share/vm/gc/g1/heapRegion.inline.hpp Mon Jul 03 14:33:00 2017 +0200 +++ b/hotspot/src/share/vm/gc/g1/heapRegion.inline.hpp Fri Aug 04 12:24:33 2017 -0700 @@ -113,7 +113,7 @@ return _bot_part.block_start_const(p); } -inline bool HeapRegion::is_obj_dead_with_size(const oop obj, G1CMBitMapRO* prev_bitmap, size_t* size) const { +inline bool HeapRegion::is_obj_dead_with_size(const oop obj, const G1CMBitMap* const prev_bitmap, size_t* size) const { HeapWord* addr = (HeapWord*) obj; assert(addr < top(), "must be"); @@ -145,7 +145,7 @@ return p < top(); } -inline size_t HeapRegion::block_size_using_bitmap(const HeapWord* addr, const G1CMBitMapRO* prev_bitmap) const { +inline size_t HeapRegion::block_size_using_bitmap(const HeapWord* addr, const G1CMBitMap* const prev_bitmap) const { assert(ClassUnloadingWithConcurrentMark, "All blocks should be objects if class unloading isn't used, so this method should not be called. " "HR: [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT ") " @@ -154,15 +154,15 @@ // Old regions' dead objects may have dead classes // We need to find the next live object using the bitmap - HeapWord* next = prev_bitmap->getNextMarkedWordAddress(addr, prev_top_at_mark_start()); + HeapWord* next = prev_bitmap->get_next_marked_addr(addr, prev_top_at_mark_start()); assert(next > addr, "must get the next live object"); return pointer_delta(next, addr); } -inline bool HeapRegion::is_obj_dead(const oop obj, const G1CMBitMapRO* prev_bitmap) const { +inline bool HeapRegion::is_obj_dead(const oop obj, const G1CMBitMap* const prev_bitmap) const { assert(is_in_reserved(obj), "Object " PTR_FORMAT " must be in region", p2i(obj)); - return !obj_allocated_since_prev_marking(obj) && !prev_bitmap->isMarked((HeapWord*)obj); + return !obj_allocated_since_prev_marking(obj) && !prev_bitmap->is_marked((HeapWord*)obj); } inline size_t HeapRegion::block_size(const HeapWord *addr) const { @@ -331,7 +331,7 @@ } #endif - G1CMBitMapRO* bitmap = g1h->concurrent_mark()->prevMarkBitMap(); + const G1CMBitMap* const bitmap = g1h->concurrent_mark()->prevMarkBitMap(); do { oop obj = oop(cur); assert(obj->is_oop(true), "Not an oop at " PTR_FORMAT, p2i(cur)); diff -r 2e49bb347cbc -r b9a409ea423b hotspot/src/share/vm/oops/array.hpp --- a/hotspot/src/share/vm/oops/array.hpp Mon Jul 03 14:33:00 2017 +0200 +++ b/hotspot/src/share/vm/oops/array.hpp Fri Aug 04 12:24:33 2017 -0700 @@ -134,10 +134,6 @@ return (int)words; } - static int size(int length, int elm_byte_size) { - return align_size_up(byte_sizeof(length, elm_byte_size), BytesPerWord) / BytesPerWord; // FIXME - } - int size() { return size(_length); }