src/hotspot/share/gc/g1/g1EvacFailure.cpp
author tschatzl
Wed, 18 Apr 2018 11:36:48 +0200
changeset 49806 2d62570a615c
parent 49722 a47d1e21b3f1
child 50752 9d62da00bf15
permissions -rw-r--r--
8200426: Make G1 code use _g1h members Summary: Consistently use _g1h member names for cached G1CollectedHeap* variables. Reviewed-by: sangheki, sjohanss
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
     1
/*
49164
7e958a8ebcd3 8195142: Refactor out card table from CardTableModRefBS to flatten the BarrierSet hierarchy
eosterlund
parents: 48969
diff changeset
     2
 * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
     3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
     4
 *
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
     5
 * This code is free software; you can redistribute it and/or modify it
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
     6
 * under the terms of the GNU General Public License version 2 only, as
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
     7
 * published by the Free Software Foundation.
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
     8
 *
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
     9
 * This code is distributed in the hope that it will be useful, but WITHOUT
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    12
 * version 2 for more details (a copy is included in the LICENSE file that
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    13
 * accompanied this code).
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    14
 *
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    15
 * You should have received a copy of the GNU General Public License version
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    16
 * 2 along with this work; if not, write to the Free Software Foundation,
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    18
 *
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    20
 * or visit www.oracle.com if you need additional information or have any
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    21
 * questions.
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    22
 *
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    23
 */
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    24
29080
a16b0d9a08f5 8072910: Move the g1EvacFailure.hpp implementation to g1EvacFailure.cpp
stefank
parents: 27009
diff changeset
    25
#include "precompiled.hpp"
30764
fec48bf5a827 8079792: GC directory structure cleanup
pliden
parents: 29080
diff changeset
    26
#include "gc/g1/dirtyCardQueue.hpp"
fec48bf5a827 8079792: GC directory structure cleanup
pliden
parents: 29080
diff changeset
    27
#include "gc/g1/g1CollectedHeap.inline.hpp"
31331
a7c714b6cfb3 7097567: G1: abstract and encapsulate collector phases and transitions between them
drwhite
parents: 30764
diff changeset
    28
#include "gc/g1/g1CollectorState.hpp"
35943
e726308008c0 8148973: Rename g1/concurrentMark.{hpp,cpp,inline.hpp} to g1/g1ConcurrentMark.{hpp,cpp,inline.hpp}
ehelin
parents: 35851
diff changeset
    29
#include "gc/g1/g1ConcurrentMark.inline.hpp"
30764
fec48bf5a827 8079792: GC directory structure cleanup
pliden
parents: 29080
diff changeset
    30
#include "gc/g1/g1EvacFailure.hpp"
35851
5451ff646f8e 8147814: Move verification code out of g1collectedheap
david
parents: 34614
diff changeset
    31
#include "gc/g1/g1HeapVerifier.hpp"
30764
fec48bf5a827 8079792: GC directory structure cleanup
pliden
parents: 29080
diff changeset
    32
#include "gc/g1/g1OopClosures.inline.hpp"
fec48bf5a827 8079792: GC directory structure cleanup
pliden
parents: 29080
diff changeset
    33
#include "gc/g1/g1_globals.hpp"
fec48bf5a827 8079792: GC directory structure cleanup
pliden
parents: 29080
diff changeset
    34
#include "gc/g1/heapRegion.hpp"
fec48bf5a827 8079792: GC directory structure cleanup
pliden
parents: 29080
diff changeset
    35
#include "gc/g1/heapRegionRemSet.hpp"
38081
a3bcb7197d45 8151556: Use the PreservedMarks* classes for the G1 preserved mark stacks
tonyp
parents: 37988
diff changeset
    36
#include "gc/shared/preservedMarks.inline.hpp"
49592
77fb0be7d19f 8199946: Move load/store and encode/decode out of oopDesc
stefank
parents: 49164
diff changeset
    37
#include "oops/access.inline.hpp"
77fb0be7d19f 8199946: Move load/store and encode/decode out of oopDesc
stefank
parents: 49164
diff changeset
    38
#include "oops/compressedOops.inline.hpp"
49722
a47d1e21b3f1 8199735: Mark word updates need to use Access API
rkennke
parents: 49643
diff changeset
    39
#include "oops/oop.inline.hpp"
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    40
46590
cd27cb97655c 8183124: Remove OopsInHeapRegionClosure
ehelin
parents: 46258
diff changeset
    41
class UpdateRSetDeferred : public ExtendedOopClosure {
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    42
private:
49806
2d62570a615c 8200426: Make G1 code use _g1h members
tschatzl
parents: 49722
diff changeset
    43
  G1CollectedHeap* _g1h;
49164
7e958a8ebcd3 8195142: Refactor out card table from CardTableModRefBS to flatten the BarrierSet hierarchy
eosterlund
parents: 48969
diff changeset
    44
  DirtyCardQueue* _dcq;
7e958a8ebcd3 8195142: Refactor out card table from CardTableModRefBS to flatten the BarrierSet hierarchy
eosterlund
parents: 48969
diff changeset
    45
  G1CardTable*    _ct;
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    46
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    47
public:
31631
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
    48
  UpdateRSetDeferred(DirtyCardQueue* dcq) :
49806
2d62570a615c 8200426: Make G1 code use _g1h members
tschatzl
parents: 49722
diff changeset
    49
    _g1h(G1CollectedHeap::heap()), _ct(_g1h->card_table()), _dcq(dcq) {}
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    50
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    51
  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    52
  virtual void do_oop(      oop* p) { do_oop_work(p); }
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    53
  template <class T> void do_oop_work(T* p) {
49806
2d62570a615c 8200426: Make G1 code use _g1h members
tschatzl
parents: 49722
diff changeset
    54
    assert(_g1h->heap_region_containing(p)->is_in_reserved(p), "paranoia");
2d62570a615c 8200426: Make G1 code use _g1h members
tschatzl
parents: 49722
diff changeset
    55
    assert(!_g1h->heap_region_containing(p)->is_survivor(), "Unexpected evac failure in survivor region");
34614
5f1e98f98bb7 8144040: UpdateRSetDeferred in G1EvacFailure will never visit survivor regions
ehelin
parents: 34311
diff changeset
    56
49592
77fb0be7d19f 8199946: Move load/store and encode/decode out of oopDesc
stefank
parents: 49164
diff changeset
    57
    T const o = RawAccess<>::oop_load(p);
77fb0be7d19f 8199946: Move load/store and encode/decode out of oopDesc
stefank
parents: 49164
diff changeset
    58
    if (CompressedOops::is_null(o)) {
46634
f0fb6b3c47d2 8183127: UpdateRSetDeferred should not enqueue cards for NULL references
tschatzl
parents: 46590
diff changeset
    59
      return;
f0fb6b3c47d2 8183127: UpdateRSetDeferred should not enqueue cards for NULL references
tschatzl
parents: 46590
diff changeset
    60
    }
f0fb6b3c47d2 8183127: UpdateRSetDeferred should not enqueue cards for NULL references
tschatzl
parents: 46590
diff changeset
    61
49592
77fb0be7d19f 8199946: Move load/store and encode/decode out of oopDesc
stefank
parents: 49164
diff changeset
    62
    if (HeapRegion::is_in_same_region(p, CompressedOops::decode(o))) {
46634
f0fb6b3c47d2 8183127: UpdateRSetDeferred should not enqueue cards for NULL references
tschatzl
parents: 46590
diff changeset
    63
      return;
f0fb6b3c47d2 8183127: UpdateRSetDeferred should not enqueue cards for NULL references
tschatzl
parents: 46590
diff changeset
    64
    }
49164
7e958a8ebcd3 8195142: Refactor out card table from CardTableModRefBS to flatten the BarrierSet hierarchy
eosterlund
parents: 48969
diff changeset
    65
    size_t card_index = _ct->index_for(p);
7e958a8ebcd3 8195142: Refactor out card table from CardTableModRefBS to flatten the BarrierSet hierarchy
eosterlund
parents: 48969
diff changeset
    66
    if (_ct->mark_card_deferred(card_index)) {
7e958a8ebcd3 8195142: Refactor out card table from CardTableModRefBS to flatten the BarrierSet hierarchy
eosterlund
parents: 48969
diff changeset
    67
      _dcq->enqueue((jbyte*)_ct->byte_for_index(card_index));
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    68
    }
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    69
  }
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    70
};
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    71
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    72
class RemoveSelfForwardPtrObjClosure: public ObjectClosure {
49806
2d62570a615c 8200426: Make G1 code use _g1h members
tschatzl
parents: 49722
diff changeset
    73
  G1CollectedHeap* _g1h;
35943
e726308008c0 8148973: Rename g1/concurrentMark.{hpp,cpp,inline.hpp} to g1/g1ConcurrentMark.{hpp,cpp,inline.hpp}
ehelin
parents: 35851
diff changeset
    74
  G1ConcurrentMark* _cm;
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    75
  HeapRegion* _hr;
11455
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
    76
  size_t _marked_bytes;
46590
cd27cb97655c 8183124: Remove OopsInHeapRegionClosure
ehelin
parents: 46258
diff changeset
    77
  UpdateRSetDeferred* _update_rset_cl;
11455
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
    78
  bool _during_initial_mark;
11583
83a7383de44c 6484965: G1: piggy-back liveness accounting phase on marking
johnc
parents: 11455
diff changeset
    79
  uint _worker_id;
31631
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
    80
  HeapWord* _last_forwarded_object_end;
11583
83a7383de44c 6484965: G1: piggy-back liveness accounting phase on marking
johnc
parents: 11455
diff changeset
    81
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    82
public:
31631
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
    83
  RemoveSelfForwardPtrObjClosure(HeapRegion* hr,
46590
cd27cb97655c 8183124: Remove OopsInHeapRegionClosure
ehelin
parents: 46258
diff changeset
    84
                                 UpdateRSetDeferred* update_rset_cl,
11455
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
    85
                                 bool during_initial_mark,
11583
83a7383de44c 6484965: G1: piggy-back liveness accounting phase on marking
johnc
parents: 11455
diff changeset
    86
                                 uint worker_id) :
49806
2d62570a615c 8200426: Make G1 code use _g1h members
tschatzl
parents: 49722
diff changeset
    87
    _g1h(G1CollectedHeap::heap()),
2d62570a615c 8200426: Make G1 code use _g1h members
tschatzl
parents: 49722
diff changeset
    88
    _cm(_g1h->concurrent_mark()),
31631
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
    89
    _hr(hr),
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
    90
    _marked_bytes(0),
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    91
    _update_rset_cl(update_rset_cl),
11455
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
    92
    _during_initial_mark(during_initial_mark),
25492
d27050bdfb04 8049421: G1 Class Unloading after completing a concurrent mark cycle
stefank
parents: 25491
diff changeset
    93
    _worker_id(worker_id),
31631
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
    94
    _last_forwarded_object_end(hr->bottom()) { }
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    95
11455
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
    96
  size_t marked_bytes() { return _marked_bytes; }
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
    97
31631
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
    98
  // Iterate over the live objects in the region to find self-forwarded objects
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
    99
  // that need to be kept live. We need to update the remembered sets of these
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   100
  // objects. Further update the BOT and marks.
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   101
  // We can coalesce and overwrite the remaining heap contents with dummy objects
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   102
  // as they have either been dead or evacuated (which are unreferenced now, i.e.
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   103
  // dead too) already.
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   104
  void do_object(oop obj) {
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   105
    HeapWord* obj_addr = (HeapWord*) obj;
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   106
    assert(_hr->is_in(obj_addr), "sanity");
25492
d27050bdfb04 8049421: G1 Class Unloading after completing a concurrent mark cycle
stefank
parents: 25491
diff changeset
   107
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   108
    if (obj->is_forwarded() && obj->forwardee() == obj) {
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   109
      // The object failed to move.
11455
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
   110
31631
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   111
      zap_dead_objects(_last_forwarded_object_end, obj_addr);
11455
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
   112
      // We consider all objects that we find self-forwarded to be
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
   113
      // live. What we'll do is that we'll update the prev marking
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
   114
      // info so that they are all under PTAMS and explicitly marked.
47678
c84eeb55c55e 8184667: Clean up G1ConcurrentMark files
tschatzl
parents: 47216
diff changeset
   115
      if (!_cm->is_marked_in_prev_bitmap(obj)) {
c84eeb55c55e 8184667: Clean up G1ConcurrentMark files
tschatzl
parents: 47216
diff changeset
   116
        _cm->mark_in_prev_bitmap(obj);
25492
d27050bdfb04 8049421: G1 Class Unloading after completing a concurrent mark cycle
stefank
parents: 25491
diff changeset
   117
      }
11455
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
   118
      if (_during_initial_mark) {
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
   119
        // For the next marking info we'll only mark the
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
   120
        // self-forwarded objects explicitly if we are during
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
   121
        // initial-mark (since, normally, we only mark objects pointed
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
   122
        // to by roots if we succeed in copying them). By marking all
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
   123
        // self-forwarded objects we ensure that we mark any that are
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
   124
        // still pointed to be roots. During concurrent marking, and
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
   125
        // after initial-mark, we don't need to mark any objects
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
   126
        // explicitly and all objects in the CSet are considered
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
   127
        // (implicitly) live. So, we won't mark them explicitly and
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
   128
        // we'll leave them over NTAMS.
49606
9ae8719efcae 8197850: Calculate liveness in regions during marking
tschatzl
parents: 49592
diff changeset
   129
        _cm->mark_in_next_bitmap(_worker_id, obj);
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   130
      }
37413
2f71679d06dd 8077144: Concurrent mark initialization takes too long
tschatzl
parents: 35943
diff changeset
   131
      size_t obj_size = obj->size();
2f71679d06dd 8077144: Concurrent mark initialization takes too long
tschatzl
parents: 35943
diff changeset
   132
11455
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
   133
      _marked_bytes += (obj_size * HeapWordSize);
38081
a3bcb7197d45 8151556: Use the PreservedMarks* classes for the G1 preserved mark stacks
tonyp
parents: 37988
diff changeset
   134
      PreservedMarks::init_forwarded_mark(obj);
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   135
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   136
      // While we were processing RSet buffers during the collection,
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   137
      // we actually didn't scan any cards on the collection set,
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   138
      // since we didn't want to update remembered sets with entries
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   139
      // that point into the collection set, given that live objects
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   140
      // from the collection set are about to move and such entries
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   141
      // will be stale very soon.
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   142
      // This change also dealt with a reliability issue which
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   143
      // involved scanning a card in the collection set and coming
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   144
      // across an array that was being chunked and looking malformed.
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   145
      // The problem is that, if evacuation fails, we might have
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   146
      // remembered set entries missing given that we skipped cards on
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   147
      // the collection set. So, we'll recreate such entries now.
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   148
      obj->oop_iterate(_update_rset_cl);
31631
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   149
37413
2f71679d06dd 8077144: Concurrent mark initialization takes too long
tschatzl
parents: 35943
diff changeset
   150
      HeapWord* obj_end = obj_addr + obj_size;
31631
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   151
      _last_forwarded_object_end = obj_end;
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   152
      _hr->cross_threshold(obj_addr, obj_end);
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   153
    }
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   154
  }
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   155
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   156
  // Fill the memory area from start to end with filler objects, and update the BOT
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   157
  // and the mark bitmap accordingly.
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   158
  void zap_dead_objects(HeapWord* start, HeapWord* end) {
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   159
    if (start == end) {
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   160
      return;
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   161
    }
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   162
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   163
    size_t gap_size = pointer_delta(end, start);
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   164
    MemRegion mr(start, gap_size);
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   165
    if (gap_size >= CollectedHeap::min_fill_size()) {
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   166
      CollectedHeap::fill_with_objects(start, gap_size);
25492
d27050bdfb04 8049421: G1 Class Unloading after completing a concurrent mark cycle
stefank
parents: 25491
diff changeset
   167
31631
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   168
      HeapWord* end_first_obj = start + ((oop)start)->size();
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   169
      _hr->cross_threshold(start, end_first_obj);
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   170
      // Fill_with_objects() may have created multiple (i.e. two)
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   171
      // objects, as the max_fill_size() is half a region.
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   172
      // After updating the BOT for the first object, also update the
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   173
      // BOT for the second object to make the BOT complete.
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   174
      if (end_first_obj != end) {
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   175
        _hr->cross_threshold(end_first_obj, end);
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   176
#ifdef ASSERT
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   177
        size_t size_second_obj = ((oop)end_first_obj)->size();
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   178
        HeapWord* end_of_second_obj = end_first_obj + size_second_obj;
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   179
        assert(end == end_of_second_obj,
33105
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 31631
diff changeset
   180
               "More than two objects were used to fill the area from " PTR_FORMAT " to " PTR_FORMAT ", "
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 31631
diff changeset
   181
               "second objects size " SIZE_FORMAT " ends at " PTR_FORMAT,
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 31631
diff changeset
   182
               p2i(start), p2i(end), size_second_obj, p2i(end_of_second_obj));
31631
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   183
#endif
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   184
      }
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   185
    }
47678
c84eeb55c55e 8184667: Clean up G1ConcurrentMark files
tschatzl
parents: 47216
diff changeset
   186
    _cm->clear_range_in_prev_bitmap(mr);
31631
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   187
  }
25492
d27050bdfb04 8049421: G1 Class Unloading after completing a concurrent mark cycle
stefank
parents: 25491
diff changeset
   188
31631
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   189
  void zap_remainder() {
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   190
    zap_dead_objects(_last_forwarded_object_end, _hr->top());
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   191
  }
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   192
};
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   193
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   194
class RemoveSelfForwardPtrHRClosure: public HeapRegionClosure {
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   195
  G1CollectedHeap* _g1h;
11583
83a7383de44c 6484965: G1: piggy-back liveness accounting phase on marking
johnc
parents: 11455
diff changeset
   196
  uint _worker_id;
27009
e7e723732b6b 8058298: Separate heap region iterator claim values from the data structures iterated over
mlarsson
parents: 26846
diff changeset
   197
  HeapRegionClaimer* _hrclaimer;
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   198
26701
f8ff74a6c058 8052172: Evacuation failure handling in G1 does not evacuate all objects if -XX:-G1DeferredRSUpdate is set
tschatzl
parents: 26422
diff changeset
   199
  DirtyCardQueue _dcq;
f8ff74a6c058 8052172: Evacuation failure handling in G1 does not evacuate all objects if -XX:-G1DeferredRSUpdate is set
tschatzl
parents: 26422
diff changeset
   200
  UpdateRSetDeferred _update_rset_cl;
f8ff74a6c058 8052172: Evacuation failure handling in G1 does not evacuate all objects if -XX:-G1DeferredRSUpdate is set
tschatzl
parents: 26422
diff changeset
   201
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   202
public:
31631
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   203
  RemoveSelfForwardPtrHRClosure(uint worker_id,
27009
e7e723732b6b 8058298: Separate heap region iterator claim values from the data structures iterated over
mlarsson
parents: 26846
diff changeset
   204
                                HeapRegionClaimer* hrclaimer) :
31631
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   205
    _g1h(G1CollectedHeap::heap()),
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   206
    _dcq(&_g1h->dirty_card_queue_set()),
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   207
    _update_rset_cl(&_dcq),
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   208
    _worker_id(worker_id),
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   209
    _hrclaimer(hrclaimer) {
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   210
  }
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   211
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   212
  size_t remove_self_forward_ptr_by_walking_hr(HeapRegion* hr,
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   213
                                               bool during_initial_mark) {
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   214
    RemoveSelfForwardPtrObjClosure rspc(hr,
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   215
                                        &_update_rset_cl,
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   216
                                        during_initial_mark,
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   217
                                        _worker_id);
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   218
    hr->object_iterate(&rspc);
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   219
    // Need to zap the remainder area of the processed region.
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   220
    rspc.zap_remainder();
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   221
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   222
    return rspc.marked_bytes();
27009
e7e723732b6b 8058298: Separate heap region iterator claim values from the data structures iterated over
mlarsson
parents: 26846
diff changeset
   223
  }
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   224
48969
jwilhelm
parents: 48890
diff changeset
   225
  bool do_heap_region(HeapRegion *hr) {
33105
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 31631
diff changeset
   226
    assert(!hr->is_pinned(), "Unexpected pinned region at index %u", hr->hrm_index());
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   227
    assert(hr->in_collection_set(), "bad CS");
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   228
27009
e7e723732b6b 8058298: Separate heap region iterator claim values from the data structures iterated over
mlarsson
parents: 26846
diff changeset
   229
    if (_hrclaimer->claim_region(hr->hrm_index())) {
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   230
      if (hr->evacuation_failed()) {
49643
a3453bbd5418 8199742: Clean up state flags in G1CollectorState
tschatzl
parents: 49607
diff changeset
   231
        bool during_initial_mark = _g1h->collector_state()->in_initial_mark_gc();
a3453bbd5418 8199742: Clean up state flags in G1CollectorState
tschatzl
parents: 49607
diff changeset
   232
        bool during_conc_mark = _g1h->collector_state()->mark_or_rebuild_in_progress();
46258
d26ebd7e2f10 8142749: HeapRegion::_predicted_bytes_to_copy is unused and can be removed
fzhinkin
parents: 39698
diff changeset
   233
11455
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
   234
        hr->note_self_forwarding_removal_start(during_initial_mark,
a6ab3d8b9a4c 6888336: G1: avoid explicitly marking and pushing objects in survivor spaces
tonyp
parents: 11451
diff changeset
   235
                                               during_conc_mark);
35851
5451ff646f8e 8147814: Move verification code out of g1collectedheap
david
parents: 34614
diff changeset
   236
        _g1h->verifier()->check_bitmaps("Self-Forwarding Ptr Removal", hr);
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   237
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   238
        hr->reset_bot();
31631
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   239
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   240
        size_t live_bytes = remove_self_forward_ptr_by_walking_hr(hr, during_initial_mark);
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   241
26422
4ee5901e205e 8048268: G1 Code Root Migration performs poorly
mgerdin
parents: 25492
diff changeset
   242
        hr->rem_set()->clean_strong_code_roots(hr);
49607
acffe6ff3ae7 8180415: Rebuild remembered sets during the concurrent cycle
tschatzl
parents: 49606
diff changeset
   243
        hr->rem_set()->clear_locked(true);
26422
4ee5901e205e 8048268: G1 Code Root Migration performs poorly
mgerdin
parents: 25492
diff changeset
   244
46258
d26ebd7e2f10 8142749: HeapRegion::_predicted_bytes_to_copy is unused and can be removed
fzhinkin
parents: 39698
diff changeset
   245
        hr->note_self_forwarding_removal_end(live_bytes);
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   246
      }
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   247
    }
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   248
    return false;
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   249
  }
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   250
};
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   251
31631
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   252
G1ParRemoveSelfForwardPtrsTask::G1ParRemoveSelfForwardPtrsTask() :
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   253
  AbstractGangTask("G1 Remove Self-forwarding Pointers"),
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   254
  _g1h(G1CollectedHeap::heap()),
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   255
  _hrclaimer(_g1h->workers()->active_workers()) { }
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   256
29080
a16b0d9a08f5 8072910: Move the g1EvacFailure.hpp implementation to g1EvacFailure.cpp
stefank
parents: 27009
diff changeset
   257
void G1ParRemoveSelfForwardPtrsTask::work(uint worker_id) {
31631
4536cf7bc89f 8129558: Coalesce dead objects during removal of self-forwarded pointers
tschatzl
parents: 31346
diff changeset
   258
  RemoveSelfForwardPtrHRClosure rsfp_cl(worker_id, &_hrclaimer);
11451
9c9aa4d85660 7121496: G1: do the per-region evacuation failure handling work in parallel
johnc
parents:
diff changeset
   259
39698
4016de4e596b 8159978: Use an array to store the collection set regions instead of linking through regions
tschatzl
parents: 38081
diff changeset
   260
  _g1h->collection_set_iterate_from(&rsfp_cl, worker_id);
29080
a16b0d9a08f5 8072910: Move the g1EvacFailure.hpp implementation to g1EvacFailure.cpp
stefank
parents: 27009
diff changeset
   261
}