hotspot/src/share/vm/gc/g1/g1ParScanThreadState.cpp
author kbarrett
Thu, 15 Oct 2015 10:10:13 -0400
changeset 33224 bc5da96c1f0e
parent 33213 b937f634f56e
child 33786 ac8da6513351
permissions -rw-r--r--
8139200: Eliminate G1ParClosureSuper::_worker_id Summary: Moved _worker_id from G1ParClosureSuper to G1ParCopyHelper. Reviewed-by: mgerdin, tschatzl
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
     1
/*
29689
dc72789f83ae 8075249: Cleanup forward_to_atomic and ClaimedForwardPtr
stefank
parents: 28213
diff changeset
     2
 * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved.
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
     3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
     4
 *
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
     5
 * This code is free software; you can redistribute it and/or modify it
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
     6
 * under the terms of the GNU General Public License version 2 only, as
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
     7
 * published by the Free Software Foundation.
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
     8
 *
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
     9
 * This code is distributed in the hope that it will be useful, but WITHOUT
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    12
 * version 2 for more details (a copy is included in the LICENSE file that
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    13
 * accompanied this code).
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    14
 *
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    15
 * You should have received a copy of the GNU General Public License version
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    16
 * 2 along with this work; if not, write to the Free Software Foundation,
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    18
 *
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    20
 * or visit www.oracle.com if you need additional information or have any
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    21
 * questions.
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    22
 *
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    23
 */
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    24
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    25
#include "precompiled.hpp"
32378
8dd0e7359751 8133470: Uses of Atomic methods in plab.hpp should be moved to .inline.hpp file
tschatzl
parents: 32377
diff changeset
    26
#include "gc/g1/g1Allocator.inline.hpp"
30764
fec48bf5a827 8079792: GC directory structure cleanup
pliden
parents: 30568
diff changeset
    27
#include "gc/g1/g1CollectedHeap.inline.hpp"
fec48bf5a827 8079792: GC directory structure cleanup
pliden
parents: 30568
diff changeset
    28
#include "gc/g1/g1OopClosures.inline.hpp"
fec48bf5a827 8079792: GC directory structure cleanup
pliden
parents: 30568
diff changeset
    29
#include "gc/g1/g1ParScanThreadState.inline.hpp"
33213
b937f634f56e 8138762: Refactor setup of evacuation closures in G1
mgerdin
parents: 33105
diff changeset
    30
#include "gc/g1/g1RootClosures.hpp"
30764
fec48bf5a827 8079792: GC directory structure cleanup
pliden
parents: 30568
diff changeset
    31
#include "gc/g1/g1StringDedup.hpp"
fec48bf5a827 8079792: GC directory structure cleanup
pliden
parents: 30568
diff changeset
    32
#include "gc/shared/taskqueue.inline.hpp"
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    33
#include "oops/oop.inline.hpp"
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    34
#include "runtime/prefetch.inline.hpp"
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    35
32737
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
    36
G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h, uint worker_id, size_t young_cset_length)
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    37
  : _g1h(g1h),
32187
0891f3fa84fc 8133047: Rename G1ParScanThreadState::_queue_num to _worker_id
tschatzl
parents: 32186
diff changeset
    38
    _refs(g1h->task_queue(worker_id)),
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    39
    _dcq(&g1h->dirty_card_queue_set()),
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    40
    _ct_bs(g1h->g1_barrier_set()),
33213
b937f634f56e 8138762: Refactor setup of evacuation closures in G1
mgerdin
parents: 33105
diff changeset
    41
    _closures(NULL),
32381
b750fd963056 8040162: Avoid reallocating PLABs between GC phases in G1
tschatzl
parents: 32379
diff changeset
    42
    _hash_seed(17),
b750fd963056 8040162: Avoid reallocating PLABs between GC phases in G1
tschatzl
parents: 32379
diff changeset
    43
    _worker_id(worker_id),
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
    44
    _tenuring_threshold(g1h->g1_policy()->tenuring_threshold()),
32381
b750fd963056 8040162: Avoid reallocating PLABs between GC phases in G1
tschatzl
parents: 32379
diff changeset
    45
    _age_table(false),
33224
bc5da96c1f0e 8139200: Eliminate G1ParClosureSuper::_worker_id
kbarrett
parents: 33213
diff changeset
    46
    _scanner(g1h, this),
32377
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
    47
    _old_gen_is_full(false)
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
    48
{
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    49
  // we allocate G1YoungSurvRateNumRegions plus one entries, since
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    50
  // we "sacrifice" entry 0 to keep track of surviving bytes for
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    51
  // non-young regions (where the age is -1)
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    52
  // We also add a few elements at the beginning and at the end in
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    53
  // an attempt to eliminate cache contention
32737
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
    54
  size_t real_length = 1 + young_cset_length;
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
    55
  size_t array_length = PADDING_ELEM_NUM +
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    56
                      real_length +
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    57
                      PADDING_ELEM_NUM;
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    58
  _surviving_young_words_base = NEW_C_HEAP_ARRAY(size_t, array_length, mtGC);
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    59
  if (_surviving_young_words_base == NULL)
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    60
    vm_exit_out_of_memory(array_length * sizeof(size_t), OOM_MALLOC_ERROR,
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    61
                          "Not enough space for young surv histo.");
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    62
  _surviving_young_words = _surviving_young_words_base + PADDING_ELEM_NUM;
32737
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
    63
  memset(_surviving_young_words, 0, real_length * sizeof(size_t));
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    64
32185
49a57ff2c3cb 8073052: Rename and clean up the allocation manager hierarchy in g1Allocator.?pp
tschatzl
parents: 31976
diff changeset
    65
  _plab_allocator = G1PLABAllocator::create_allocator(_g1h->allocator());
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    66
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
    67
  _dest[InCSetState::NotInCSet]    = InCSetState::NotInCSet;
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
    68
  // The dest for Young is used when the objects are aged enough to
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
    69
  // need to be moved to the next space.
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
    70
  _dest[InCSetState::Young]        = InCSetState::Old;
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
    71
  _dest[InCSetState::Old]          = InCSetState::Old;
33213
b937f634f56e 8138762: Refactor setup of evacuation closures in G1
mgerdin
parents: 33105
diff changeset
    72
b937f634f56e 8138762: Refactor setup of evacuation closures in G1
mgerdin
parents: 33105
diff changeset
    73
  _closures = G1EvacuationRootClosures::create_root_closures(this, _g1h);
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    74
}
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    75
32736
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
    76
// Pass locally gathered statistics to global state.
32737
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
    77
void G1ParScanThreadState::flush(size_t* surviving_young_words) {
32736
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
    78
  _dcq.flush();
32382
8ab915777257 8073146: G1 merges thread local age tables too early with global age table
tschatzl
parents: 32381
diff changeset
    79
  // Update allocation statistics.
32379
aa14adafaf0f 8073013: Add detailed information about PLAB memory usage
tschatzl
parents: 32378
diff changeset
    80
  _plab_allocator->flush_and_retire_stats();
32736
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
    81
  _g1h->g1_policy()->record_age_table(&_age_table);
32737
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
    82
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
    83
  uint length = _g1h->g1_policy()->young_cset_region_length();
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
    84
  for (uint region_index = 0; region_index < length; region_index++) {
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
    85
    surviving_young_words[region_index] += _surviving_young_words[region_index];
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
    86
  }
32736
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
    87
}
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
    88
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
    89
G1ParScanThreadState::~G1ParScanThreadState() {
32185
49a57ff2c3cb 8073052: Rename and clean up the allocation manager hierarchy in g1Allocator.?pp
tschatzl
parents: 31976
diff changeset
    90
  delete _plab_allocator;
33213
b937f634f56e 8138762: Refactor setup of evacuation closures in G1
mgerdin
parents: 33105
diff changeset
    91
  delete _closures;
27880
afb974a04396 8060074: os::free() takes MemoryTrackingLevel but doesn't need it
coleenp
parents: 26837
diff changeset
    92
  FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_base);
25483
962ccf95c515 8035401: Fix visibility of G1ParScanThreadState members
tschatzl
parents: 25482
diff changeset
    93
}
962ccf95c515 8035401: Fix visibility of G1ParScanThreadState members
tschatzl
parents: 25482
diff changeset
    94
32381
b750fd963056 8040162: Avoid reallocating PLABs between GC phases in G1
tschatzl
parents: 32379
diff changeset
    95
void G1ParScanThreadState::waste(size_t& wasted, size_t& undo_wasted) {
b750fd963056 8040162: Avoid reallocating PLABs between GC phases in G1
tschatzl
parents: 32379
diff changeset
    96
  _plab_allocator->waste(wasted, undo_wasted);
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    97
}
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    98
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
    99
#ifdef ASSERT
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   100
bool G1ParScanThreadState::verify_ref(narrowOop* ref) const {
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   101
  assert(ref != NULL, "invariant");
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   102
  assert(UseCompressedOops, "sanity");
33105
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 32737
diff changeset
   103
  assert(!has_partial_array_mask(ref), "ref=" PTR_FORMAT, p2i(ref));
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   104
  oop p = oopDesc::load_decode_heap_oop(ref);
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   105
  assert(_g1h->is_in_g1_reserved(p),
33105
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 32737
diff changeset
   106
         "ref=" PTR_FORMAT " p=" PTR_FORMAT, p2i(ref), p2i(p));
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   107
  return true;
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   108
}
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   109
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   110
bool G1ParScanThreadState::verify_ref(oop* ref) const {
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   111
  assert(ref != NULL, "invariant");
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   112
  if (has_partial_array_mask(ref)) {
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   113
    // Must be in the collection set--it's already been copied.
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   114
    oop p = clear_partial_array_mask(ref);
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   115
    assert(_g1h->obj_in_cs(p),
33105
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 32737
diff changeset
   116
           "ref=" PTR_FORMAT " p=" PTR_FORMAT, p2i(ref), p2i(p));
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   117
  } else {
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   118
    oop p = oopDesc::load_decode_heap_oop(ref);
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   119
    assert(_g1h->is_in_g1_reserved(p),
33105
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 32737
diff changeset
   120
           "ref=" PTR_FORMAT " p=" PTR_FORMAT, p2i(ref), p2i(p));
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   121
  }
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   122
  return true;
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   123
}
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   124
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   125
bool G1ParScanThreadState::verify_task(StarTask ref) const {
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   126
  if (ref.is_narrow()) {
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   127
    return verify_ref((narrowOop*) ref);
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   128
  } else {
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   129
    return verify_ref((oop*) ref);
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   130
  }
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   131
}
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   132
#endif // ASSERT
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   133
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   134
void G1ParScanThreadState::trim_queue() {
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   135
  StarTask ref;
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   136
  do {
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   137
    // Drain the overflow stack first, so other threads can steal.
25483
962ccf95c515 8035401: Fix visibility of G1ParScanThreadState members
tschatzl
parents: 25482
diff changeset
   138
    while (_refs->pop_overflow(ref)) {
962ccf95c515 8035401: Fix visibility of G1ParScanThreadState members
tschatzl
parents: 25482
diff changeset
   139
      dispatch_reference(ref);
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   140
    }
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   141
25483
962ccf95c515 8035401: Fix visibility of G1ParScanThreadState members
tschatzl
parents: 25482
diff changeset
   142
    while (_refs->pop_local(ref)) {
962ccf95c515 8035401: Fix visibility of G1ParScanThreadState members
tschatzl
parents: 25482
diff changeset
   143
      dispatch_reference(ref);
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   144
    }
25483
962ccf95c515 8035401: Fix visibility of G1ParScanThreadState members
tschatzl
parents: 25482
diff changeset
   145
  } while (!_refs->is_empty());
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   146
}
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   147
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   148
HeapWord* G1ParScanThreadState::allocate_in_next_plab(InCSetState const state,
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   149
                                                      InCSetState* dest,
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   150
                                                      size_t word_sz,
32377
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   151
                                                      AllocationContext_t const context,
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   152
                                                      bool previous_plab_refill_failed) {
33105
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 32737
diff changeset
   153
  assert(state.is_in_cset_or_humongous(), "Unexpected state: " CSETSTATE_FORMAT, state.value());
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 32737
diff changeset
   154
  assert(dest->is_in_cset_or_humongous(), "Unexpected dest: " CSETSTATE_FORMAT, dest->value());
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   155
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   156
  // Right now we only have two types of regions (young / old) so
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   157
  // let's keep the logic here simple. We can generalize it when necessary.
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   158
  if (dest->is_young()) {
32377
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   159
    bool plab_refill_in_old_failed = false;
32185
49a57ff2c3cb 8073052: Rename and clean up the allocation manager hierarchy in g1Allocator.?pp
tschatzl
parents: 31976
diff changeset
   160
    HeapWord* const obj_ptr = _plab_allocator->allocate(InCSetState::Old,
49a57ff2c3cb 8073052: Rename and clean up the allocation manager hierarchy in g1Allocator.?pp
tschatzl
parents: 31976
diff changeset
   161
                                                        word_sz,
32377
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   162
                                                        context,
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   163
                                                        &plab_refill_in_old_failed);
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   164
    // Make sure that we won't attempt to copy any other objects out
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   165
    // of a survivor region (given that apparently we cannot allocate
32377
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   166
    // any new ones) to avoid coming into this slow path again and again.
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   167
    // Only consider failed PLAB refill here: failed inline allocations are
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   168
    // typically large, so not indicative of remaining space.
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   169
    if (previous_plab_refill_failed) {
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   170
      _tenuring_threshold = 0;
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   171
    }
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   172
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   173
    if (obj_ptr != NULL) {
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   174
      dest->set_old();
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   175
    } else {
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   176
      // We just failed to allocate in old gen. The same idea as explained above
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   177
      // for making survivor gen unavailable for allocation applies for old gen.
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   178
      _old_gen_is_full = plab_refill_in_old_failed;
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   179
    }
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   180
    return obj_ptr;
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   181
  } else {
32377
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   182
    _old_gen_is_full = previous_plab_refill_failed;
33105
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 32737
diff changeset
   183
    assert(dest->is_old(), "Unexpected dest: " CSETSTATE_FORMAT, dest->value());
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   184
    // no other space to try.
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   185
    return NULL;
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   186
  }
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   187
}
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   188
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   189
InCSetState G1ParScanThreadState::next_state(InCSetState const state, markOop const m, uint& age) {
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   190
  if (state.is_young()) {
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   191
    age = !m->has_displaced_mark_helper() ? m->age()
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   192
                                          : m->displaced_mark_helper()->age();
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   193
    if (age < _tenuring_threshold) {
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   194
      return state;
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   195
    }
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   196
  }
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   197
  return dest(state);
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   198
}
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   199
32608
ef2ec6fac731 8066443: Add G1 support for promotion event
sfriberg
parents: 32382
diff changeset
   200
void G1ParScanThreadState::report_promotion_event(InCSetState const dest_state,
ef2ec6fac731 8066443: Add G1 support for promotion event
sfriberg
parents: 32382
diff changeset
   201
                                                  oop const old, size_t word_sz, uint age,
ef2ec6fac731 8066443: Add G1 support for promotion event
sfriberg
parents: 32382
diff changeset
   202
                                                  HeapWord * const obj_ptr,
ef2ec6fac731 8066443: Add G1 support for promotion event
sfriberg
parents: 32382
diff changeset
   203
                                                  const AllocationContext_t context) const {
ef2ec6fac731 8066443: Add G1 support for promotion event
sfriberg
parents: 32382
diff changeset
   204
  G1PLAB* alloc_buf = _plab_allocator->alloc_buffer(dest_state, context);
ef2ec6fac731 8066443: Add G1 support for promotion event
sfriberg
parents: 32382
diff changeset
   205
  if (alloc_buf->contains(obj_ptr)) {
ef2ec6fac731 8066443: Add G1 support for promotion event
sfriberg
parents: 32382
diff changeset
   206
    _g1h->_gc_tracer_stw->report_promotion_in_new_plab_event(old->klass(), word_sz, age,
ef2ec6fac731 8066443: Add G1 support for promotion event
sfriberg
parents: 32382
diff changeset
   207
                                                             dest_state.value() == InCSetState::Old,
ef2ec6fac731 8066443: Add G1 support for promotion event
sfriberg
parents: 32382
diff changeset
   208
                                                             alloc_buf->word_sz());
ef2ec6fac731 8066443: Add G1 support for promotion event
sfriberg
parents: 32382
diff changeset
   209
  } else {
ef2ec6fac731 8066443: Add G1 support for promotion event
sfriberg
parents: 32382
diff changeset
   210
    _g1h->_gc_tracer_stw->report_promotion_outside_plab_event(old->klass(), word_sz, age,
ef2ec6fac731 8066443: Add G1 support for promotion event
sfriberg
parents: 32382
diff changeset
   211
                                                              dest_state.value() == InCSetState::Old);
ef2ec6fac731 8066443: Add G1 support for promotion event
sfriberg
parents: 32382
diff changeset
   212
  }
ef2ec6fac731 8066443: Add G1 support for promotion event
sfriberg
parents: 32382
diff changeset
   213
}
ef2ec6fac731 8066443: Add G1 support for promotion event
sfriberg
parents: 32382
diff changeset
   214
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   215
oop G1ParScanThreadState::copy_to_survivor_space(InCSetState const state,
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   216
                                                 oop const old,
27682
dbd1c3f92130 8064473: Improved handling of age during object copy in G1
sfriberg
parents: 26837
diff changeset
   217
                                                 markOop const old_mark) {
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   218
  const size_t word_sz = old->size();
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   219
  HeapRegion* const from_region = _g1h->heap_region_containing_raw(old);
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   220
  // +1 to make the -1 indexes valid...
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   221
  const int young_index = from_region->young_index_in_cset()+1;
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   222
  assert( (from_region->is_young() && young_index >  0) ||
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   223
         (!from_region->is_young() && young_index == 0), "invariant" );
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   224
  const AllocationContext_t context = from_region->allocation_context();
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   225
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   226
  uint age = 0;
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   227
  InCSetState dest_state = next_state(state, old_mark, age);
32377
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   228
  // The second clause is to prevent premature evacuation failure in case there
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   229
  // is still space in survivor, but old gen is full.
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   230
  if (_old_gen_is_full && dest_state.is_old()) {
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   231
    return handle_evacuation_failure_par(old, old_mark);
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   232
  }
32185
49a57ff2c3cb 8073052: Rename and clean up the allocation manager hierarchy in g1Allocator.?pp
tschatzl
parents: 31976
diff changeset
   233
  HeapWord* obj_ptr = _plab_allocator->plab_allocate(dest_state, word_sz, context);
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   234
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   235
  // PLAB allocations should succeed most of the time, so we'll
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   236
  // normally check against NULL once and that's it.
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   237
  if (obj_ptr == NULL) {
32377
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   238
    bool plab_refill_failed = false;
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   239
    obj_ptr = _plab_allocator->allocate_direct_or_new_plab(dest_state, word_sz, context, &plab_refill_failed);
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   240
    if (obj_ptr == NULL) {
32377
5ee15c417d02 8003237: G1: Reduce unnecessary (and failing) allocation attempts when handling an evacuation failure
tschatzl
parents: 32187
diff changeset
   241
      obj_ptr = allocate_in_next_plab(state, &dest_state, word_sz, context, plab_refill_failed);
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   242
      if (obj_ptr == NULL) {
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   243
        // This will either forward-to-self, or detect that someone else has
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   244
        // installed a forwarding pointer.
31976
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   245
        return handle_evacuation_failure_par(old, old_mark);
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   246
      }
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   247
    }
32608
ef2ec6fac731 8066443: Add G1 support for promotion event
sfriberg
parents: 32382
diff changeset
   248
    if (_g1h->_gc_tracer_stw->should_report_promotion_events()) {
ef2ec6fac731 8066443: Add G1 support for promotion event
sfriberg
parents: 32382
diff changeset
   249
      // The events are checked individually as part of the actual commit
ef2ec6fac731 8066443: Add G1 support for promotion event
sfriberg
parents: 32382
diff changeset
   250
      report_promotion_event(dest_state, old, word_sz, age, obj_ptr, context);
ef2ec6fac731 8066443: Add G1 support for promotion event
sfriberg
parents: 32382
diff changeset
   251
    }
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   252
  }
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   253
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   254
  assert(obj_ptr != NULL, "when we get here, allocation should have succeeded");
29689
dc72789f83ae 8075249: Cleanup forward_to_atomic and ClaimedForwardPtr
stefank
parents: 28213
diff changeset
   255
  assert(_g1h->is_in_reserved(obj_ptr), "Allocated memory should be in the heap");
dc72789f83ae 8075249: Cleanup forward_to_atomic and ClaimedForwardPtr
stefank
parents: 28213
diff changeset
   256
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   257
#ifndef PRODUCT
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   258
  // Should this evacuation fail?
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   259
  if (_g1h->evacuation_should_fail()) {
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   260
    // Doing this after all the allocation attempts also tests the
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   261
    // undo_allocation() method too.
32185
49a57ff2c3cb 8073052: Rename and clean up the allocation manager hierarchy in g1Allocator.?pp
tschatzl
parents: 31976
diff changeset
   262
    _plab_allocator->undo_allocation(dest_state, obj_ptr, word_sz, context);
31976
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   263
    return handle_evacuation_failure_par(old, old_mark);
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   264
  }
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   265
#endif // !PRODUCT
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   266
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   267
  // We're going to allocate linearly, so might as well prefetch ahead.
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   268
  Prefetch::write(obj_ptr, PrefetchCopyIntervalInBytes);
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   269
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   270
  const oop obj = oop(obj_ptr);
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   271
  const oop forward_ptr = old->forward_to_atomic(obj);
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   272
  if (forward_ptr == NULL) {
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   273
    Copy::aligned_disjoint_words((HeapWord*) old, obj_ptr, word_sz);
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   274
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   275
    if (dest_state.is_young()) {
27682
dbd1c3f92130 8064473: Improved handling of age during object copy in G1
sfriberg
parents: 26837
diff changeset
   276
      if (age < markOopDesc::max_age) {
dbd1c3f92130 8064473: Improved handling of age during object copy in G1
sfriberg
parents: 26837
diff changeset
   277
        age++;
dbd1c3f92130 8064473: Improved handling of age during object copy in G1
sfriberg
parents: 26837
diff changeset
   278
      }
dbd1c3f92130 8064473: Improved handling of age during object copy in G1
sfriberg
parents: 26837
diff changeset
   279
      if (old_mark->has_displaced_mark_helper()) {
dbd1c3f92130 8064473: Improved handling of age during object copy in G1
sfriberg
parents: 26837
diff changeset
   280
        // In this case, we have to install the mark word first,
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   281
        // otherwise obj looks to be forwarded (the old mark word,
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   282
        // which contains the forward pointer, was copied)
27682
dbd1c3f92130 8064473: Improved handling of age during object copy in G1
sfriberg
parents: 26837
diff changeset
   283
        obj->set_mark(old_mark);
dbd1c3f92130 8064473: Improved handling of age during object copy in G1
sfriberg
parents: 26837
diff changeset
   284
        markOop new_mark = old_mark->displaced_mark_helper()->set_age(age);
dbd1c3f92130 8064473: Improved handling of age during object copy in G1
sfriberg
parents: 26837
diff changeset
   285
        old_mark->set_displaced_mark_helper(new_mark);
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   286
      } else {
27682
dbd1c3f92130 8064473: Improved handling of age during object copy in G1
sfriberg
parents: 26837
diff changeset
   287
        obj->set_mark(old_mark->set_age(age));
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   288
      }
32382
8ab915777257 8073146: G1 merges thread local age tables too early with global age table
tschatzl
parents: 32381
diff changeset
   289
      _age_table.add(age, word_sz);
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   290
    } else {
27682
dbd1c3f92130 8064473: Improved handling of age during object copy in G1
sfriberg
parents: 26837
diff changeset
   291
      obj->set_mark(old_mark);
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   292
    }
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   293
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   294
    if (G1StringDedup::is_enabled()) {
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   295
      const bool is_from_young = state.is_young();
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   296
      const bool is_to_young = dest_state.is_young();
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   297
      assert(is_from_young == _g1h->heap_region_containing_raw(old)->is_young(),
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   298
             "sanity");
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   299
      assert(is_to_young == _g1h->heap_region_containing_raw(obj)->is_young(),
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   300
             "sanity");
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   301
      G1StringDedup::enqueue_from_evacuation(is_from_young,
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   302
                                             is_to_young,
32187
0891f3fa84fc 8133047: Rename G1ParScanThreadState::_queue_num to _worker_id
tschatzl
parents: 32186
diff changeset
   303
                                             _worker_id,
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   304
                                             obj);
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   305
    }
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   306
32381
b750fd963056 8040162: Avoid reallocating PLABs between GC phases in G1
tschatzl
parents: 32379
diff changeset
   307
    _surviving_young_words[young_index] += word_sz;
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   308
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   309
    if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) {
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   310
      // We keep track of the next start index in the length field of
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   311
      // the to-space object. The actual length can be found in the
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   312
      // length field of the from-space object.
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   313
      arrayOop(obj)->set_length(0);
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   314
      oop* old_p = set_partial_array_mask(old);
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   315
      push_on_queue(old_p);
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   316
    } else {
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   317
      HeapRegion* const to_region = _g1h->heap_region_containing_raw(obj_ptr);
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   318
      _scanner.set_region(to_region);
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   319
      obj->oop_iterate_backwards(&_scanner);
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   320
    }
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   321
    return obj;
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   322
  } else {
32185
49a57ff2c3cb 8073052: Rename and clean up the allocation manager hierarchy in g1Allocator.?pp
tschatzl
parents: 31976
diff changeset
   323
    _plab_allocator->undo_allocation(dest_state, obj_ptr, word_sz, context);
28213
b0bf57cd1e9d 8060025: Object copy time regressions after JDK-8031323 and JDK-8057536
tschatzl
parents: 27883
diff changeset
   324
    return forward_ptr;
25482
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   325
  }
b69656f26643 8035400: Move G1ParScanThreadState into its own files
tschatzl
parents:
diff changeset
   326
}
31976
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   327
32736
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
   328
G1ParScanThreadState* G1ParScanThreadStateSet::state_for_worker(uint worker_id) {
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
   329
  assert(worker_id < _n_workers, "out of bounds access");
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
   330
  return _states[worker_id];
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
   331
}
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
   332
32737
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
   333
void G1ParScanThreadStateSet::add_cards_scanned(uint worker_id, size_t cards_scanned) {
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
   334
  assert(worker_id < _n_workers, "out of bounds access");
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
   335
  _cards_scanned[worker_id] += cards_scanned;
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
   336
}
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
   337
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
   338
size_t G1ParScanThreadStateSet::total_cards_scanned() const {
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
   339
  assert(_flushed, "thread local state from the per thread states should have been flushed");
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
   340
  return _total_cards_scanned;
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
   341
}
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
   342
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
   343
const size_t* G1ParScanThreadStateSet::surviving_young_words() const {
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
   344
  assert(_flushed, "thread local state from the per thread states should have been flushed");
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
   345
  return _surviving_young_words_total;
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
   346
}
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
   347
32736
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
   348
void G1ParScanThreadStateSet::flush() {
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
   349
  assert(!_flushed, "thread local state from the per thread states should be flushed once");
32737
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
   350
  assert(_total_cards_scanned == 0, "should have been cleared");
32736
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
   351
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
   352
  for (uint worker_index = 0; worker_index < _n_workers; ++worker_index) {
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
   353
    G1ParScanThreadState* pss = _states[worker_index];
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
   354
32737
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
   355
    _total_cards_scanned += _cards_scanned[worker_index];
32736
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
   356
32737
f02118695c2f 8135154: Move cards scanned and surviving young words aggregation to G1ParScanThreadStateSet
mgerdin
parents: 32736
diff changeset
   357
    pss->flush(_surviving_young_words_total);
32736
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
   358
    delete pss;
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
   359
    _states[worker_index] = NULL;
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
   360
  }
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
   361
  _flushed = true;
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
   362
}
755024a84282 8135152: Create a G1ParScanThreadStateSet class for managing G1 GC per thread states
mgerdin
parents: 32608
diff changeset
   363
31976
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   364
oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markOop m) {
33105
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 32737
diff changeset
   365
  assert(_g1h->obj_in_cs(old), "Object " PTR_FORMAT " should be in the CSet", p2i(old));
31976
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   366
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   367
  oop forward_ptr = old->forward_to_atomic(old);
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   368
  if (forward_ptr == NULL) {
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   369
    // Forward-to-self succeeded. We are the "owner" of the object.
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   370
    HeapRegion* r = _g1h->heap_region_containing(old);
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   371
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   372
    if (!r->evacuation_failed()) {
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   373
      r->set_evacuation_failed(true);
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   374
     _g1h->hr_printer()->evac_failure(r);
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   375
    }
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   376
32187
0891f3fa84fc 8133047: Rename G1ParScanThreadState::_queue_num to _worker_id
tschatzl
parents: 32186
diff changeset
   377
    _g1h->preserve_mark_during_evac_failure(_worker_id, old, m);
31976
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   378
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   379
    _scanner.set_region(r);
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   380
    old->oop_iterate_backwards(&_scanner);
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   381
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   382
    return old;
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   383
  } else {
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   384
    // Forward-to-self failed. Either someone else managed to allocate
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   385
    // space for this object (old != forward_ptr) or they beat us in
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   386
    // self-forwarding it (old == forward_ptr).
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   387
    assert(old == forward_ptr || !_g1h->obj_in_cs(forward_ptr),
33105
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 32737
diff changeset
   388
           "Object " PTR_FORMAT " forwarded to: " PTR_FORMAT " "
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 32737
diff changeset
   389
           "should not be in the CSet",
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 32737
diff changeset
   390
           p2i(old), p2i(forward_ptr));
31976
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   391
    return forward_ptr;
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   392
  }
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   393
}
da45f85bf4e1 8004687: G1: Parallelize object self-forwarding and scanning during an evacuation failure
tschatzl
parents: 30764
diff changeset
   394