src/hotspot/share/gc/z/zRelocate.cpp
author pliden
Mon, 18 Mar 2019 11:50:39 +0100
changeset 54162 f344a0c6e19e
parent 54161 349843ebb209
child 54163 790679f86a51
permissions -rw-r--r--
8220587: ZGC: Break out forwarding information from ZPage Reviewed-by: stefank, eosterlund
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     1
/*
53842
c459deff5939 8219331: ZGC: Unify TLAB retire/remap handling
pliden
parents: 51818
diff changeset
     2
 * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     4
 *
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     5
 * This code is free software; you can redistribute it and/or modify it
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     6
 * under the terms of the GNU General Public License version 2 only, as
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     7
 * published by the Free Software Foundation.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     8
 *
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     9
 * This code is distributed in the hope that it will be useful, but WITHOUT
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    12
 * version 2 for more details (a copy is included in the LICENSE file that
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    13
 * accompanied this code).
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    14
 *
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    15
 * You should have received a copy of the GNU General Public License version
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    16
 * 2 along with this work; if not, write to the Free Software Foundation,
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    18
 *
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    20
 * or visit www.oracle.com if you need additional information or have any
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    21
 * questions.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    22
 */
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    23
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    24
#include "precompiled.hpp"
51818
75e4ce0fa1ba 8210883: ZGC: Parallel retire/resize/remap of TLABs
pliden
parents: 51803
diff changeset
    25
#include "gc/z/zAddress.inline.hpp"
75e4ce0fa1ba 8210883: ZGC: Parallel retire/resize/remap of TLABs
pliden
parents: 51803
diff changeset
    26
#include "gc/z/zBarrier.inline.hpp"
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
    27
#include "gc/z/zForwarding.inline.hpp"
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    28
#include "gc/z/zHeap.hpp"
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    29
#include "gc/z/zOopClosures.inline.hpp"
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    30
#include "gc/z/zPage.hpp"
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    31
#include "gc/z/zRelocate.hpp"
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    32
#include "gc/z/zRelocationSet.inline.hpp"
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    33
#include "gc/z/zRootsIterator.hpp"
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    34
#include "gc/z/zStat.hpp"
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    35
#include "gc/z/zTask.hpp"
53842
c459deff5939 8219331: ZGC: Unify TLAB retire/remap handling
pliden
parents: 51818
diff changeset
    36
#include "gc/z/zThreadLocalAllocBuffer.hpp"
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    37
#include "gc/z/zWorkers.hpp"
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    38
#include "logging/log.hpp"
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    39
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    40
static const ZStatCounter ZCounterRelocationContention("Contention", "Relocation Contention", ZStatUnitOpsPerSecond);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    41
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    42
ZRelocate::ZRelocate(ZWorkers* workers) :
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    43
    _workers(workers) {}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    44
51803
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    45
class ZRelocateRootsIteratorClosure : public ZRootsIteratorClosure {
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    46
public:
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    47
  virtual void do_thread(Thread* thread) {
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    48
    // Update thread local address bad mask
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    49
    ZThreadLocalData::set_address_bad_mask(thread, ZAddressBadMask);
51818
75e4ce0fa1ba 8210883: ZGC: Parallel retire/resize/remap of TLABs
pliden
parents: 51803
diff changeset
    50
75e4ce0fa1ba 8210883: ZGC: Parallel retire/resize/remap of TLABs
pliden
parents: 51803
diff changeset
    51
    // Remap TLAB
53842
c459deff5939 8219331: ZGC: Unify TLAB retire/remap handling
pliden
parents: 51818
diff changeset
    52
    ZThreadLocalAllocBuffer::remap(thread);
51803
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    53
  }
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    54
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    55
  virtual void do_oop(oop* p) {
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    56
    ZBarrier::relocate_barrier_on_root_oop_field(p);
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    57
  }
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    58
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    59
  virtual void do_oop(narrowOop* p) {
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    60
    ShouldNotReachHere();
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    61
  }
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    62
};
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    63
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    64
class ZRelocateRootsTask : public ZTask {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    65
private:
51818
75e4ce0fa1ba 8210883: ZGC: Parallel retire/resize/remap of TLABs
pliden
parents: 51803
diff changeset
    66
  ZRootsIterator                _roots;
75e4ce0fa1ba 8210883: ZGC: Parallel retire/resize/remap of TLABs
pliden
parents: 51803
diff changeset
    67
  ZRelocateRootsIteratorClosure _cl;
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    68
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    69
public:
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    70
  ZRelocateRootsTask() :
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    71
      ZTask("ZRelocateRootsTask"),
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    72
      _roots() {}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    73
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    74
  virtual void work() {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    75
    // During relocation we need to visit the JVMTI
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    76
    // export weak roots to rehash the JVMTI tag map
51818
75e4ce0fa1ba 8210883: ZGC: Parallel retire/resize/remap of TLABs
pliden
parents: 51803
diff changeset
    77
    _roots.oops_do(&_cl, true /* visit_jvmti_weak_export */);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    78
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    79
};
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    80
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    81
void ZRelocate::start() {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    82
  ZRelocateRootsTask task;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    83
  _workers->run_parallel(&task);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    84
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    85
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
    86
ZForwarding* ZRelocate::forwarding_for_page(ZPage* page) const {
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
    87
  const uintptr_t addr = ZAddress::good(page->start());
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
    88
  return ZHeap::heap()->forwarding(addr);
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
    89
}
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    90
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
    91
uintptr_t ZRelocate::relocate_object_inner(ZForwarding* forwarding, uintptr_t from_index, uintptr_t from_offset) const {
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
    92
  ZForwardingCursor cursor;
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
    93
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
    94
  // Lookup forwarding entry
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
    95
  const ZForwardingEntry entry = forwarding->find(from_index, &cursor);
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    96
  if (entry.from_index() == from_index) {
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    97
    // Already relocated, return new address
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    98
    return entry.to_offset();
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    99
  }
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   100
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   101
  assert(ZHeap::heap()->is_object_live(ZAddress::good(from_offset)), "Should be live");
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   102
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   103
  if (forwarding->is_pinned()) {
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   104
    // In-place forward
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   105
    return forwarding->insert(from_index, from_offset, &cursor);
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   106
  }
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   107
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   108
  // Allocate object
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   109
  const uintptr_t from_good = ZAddress::good(from_offset);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   110
  const size_t size = ZUtils::object_size(from_good);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   111
  const uintptr_t to_good = ZHeap::heap()->alloc_object_for_relocation(size);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   112
  if (to_good == 0) {
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   113
    // Failed, in-place forward
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   114
    return forwarding->insert(from_index, from_offset, &cursor);
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   115
  }
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   116
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   117
  // Copy object
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   118
  ZUtils::object_copy(from_good, to_good, size);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   119
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   120
  // Insert forwarding entry
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   121
  const uintptr_t to_offset = ZAddress::offset(to_good);
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   122
  const uintptr_t to_offset_final = forwarding->insert(from_index, to_offset, &cursor);
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   123
  if (to_offset_final == to_offset) {
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   124
    // Relocation succeeded
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   125
    return to_offset;
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   126
  }
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   127
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   128
  // Relocation contention
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   129
  ZStatInc(ZCounterRelocationContention);
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   130
  log_trace(gc)("Relocation contention, thread: " PTR_FORMAT " (%s), forwarding: " PTR_FORMAT
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   131
                ", entry: " UINT32_FORMAT ", oop: " PTR_FORMAT ", size: " SIZE_FORMAT,
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   132
                ZThread::id(), ZThread::name(), p2i(forwarding), cursor, from_good, size);
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   133
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   134
  // Try undo allocation
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   135
  ZHeap::heap()->undo_alloc_object_for_relocation(to_good, size);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   136
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   137
  return to_offset_final;
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   138
}
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   139
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   140
uintptr_t ZRelocate::relocate_object(ZForwarding* forwarding, uintptr_t from_addr) const {
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   141
  const uintptr_t from_offset = ZAddress::offset(from_addr);
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   142
  const uintptr_t from_index = (from_offset - forwarding->start()) >> forwarding->object_alignment_shift();
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   143
  const uintptr_t to_offset = relocate_object_inner(forwarding, from_index, from_offset);
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   144
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   145
  if (from_offset == to_offset) {
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   146
    // In-place forwarding, pin page
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   147
    forwarding->set_pinned();
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   148
  }
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   149
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   150
  return ZAddress::good(to_offset);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   151
}
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   152
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   153
uintptr_t ZRelocate::forward_object(ZForwarding* forwarding, uintptr_t from_addr) const {
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   154
  const uintptr_t from_offset = ZAddress::offset(from_addr);
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   155
  const uintptr_t from_index = (from_offset - forwarding->start()) >> forwarding->object_alignment_shift();
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   156
  const ZForwardingEntry entry = forwarding->find(from_index);
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   157
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   158
  assert(entry.from_index() == from_index, "Should be forwarded");
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   159
  return ZAddress::good(entry.to_offset());
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   160
}
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   161
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   162
class ZRelocateObjectClosure : public ObjectClosure {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   163
private:
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   164
  ZRelocate* const   _relocate;
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   165
  ZForwarding* const _forwarding;
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   166
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   167
public:
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   168
  ZRelocateObjectClosure(ZRelocate* relocate, ZForwarding* forwarding) :
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   169
      _relocate(relocate),
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   170
      _forwarding(forwarding) {}
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   171
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   172
  virtual void do_object(oop o) {
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   173
    _relocate->relocate_object(_forwarding, ZOop::to_address(o));
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   174
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   175
};
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   176
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   177
bool ZRelocate::work(ZRelocationSetParallelIterator* iter) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   178
  bool success = true;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   179
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   180
  // Relocate pages in the relocation set
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   181
  for (ZPage* page; iter->next(&page);) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   182
    // Relocate objects in page
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   183
    ZForwarding* const forwarding = forwarding_for_page(page);
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   184
    ZRelocateObjectClosure cl(this, forwarding);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   185
    page->object_iterate(&cl);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   186
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   187
    if (ZVerifyForwarding) {
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   188
      forwarding->verify(page->object_max_count(), page->live_objects());
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   189
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   190
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   191
    if (forwarding->is_pinned()) {
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   192
      // Relocation failed, page is now pinned
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   193
      success = false;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   194
    } else {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   195
      // Relocation succeeded, release page
54162
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   196
      if (forwarding->dec_refcount()) {
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   197
        ZHeap::heap()->free_page(page, true /* reclaimed */);
f344a0c6e19e 8220587: ZGC: Break out forwarding information from ZPage
pliden
parents: 54161
diff changeset
   198
      }
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   199
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   200
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   201
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   202
  return success;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   203
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   204
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   205
class ZRelocateTask : public ZTask {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   206
private:
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   207
  ZRelocate* const               _relocate;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   208
  ZRelocationSetParallelIterator _iter;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   209
  bool                           _failed;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   210
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   211
public:
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   212
  ZRelocateTask(ZRelocate* relocate, ZRelocationSet* relocation_set) :
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   213
      ZTask("ZRelocateTask"),
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   214
      _relocate(relocate),
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   215
      _iter(relocation_set),
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   216
      _failed(false) {}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   217
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   218
  virtual void work() {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   219
    if (!_relocate->work(&_iter)) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   220
      _failed = true;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   221
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   222
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   223
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   224
  bool failed() const {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   225
    return _failed;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   226
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   227
};
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   228
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   229
bool ZRelocate::relocate(ZRelocationSet* relocation_set) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   230
  ZRelocateTask task(this, relocation_set);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   231
  _workers->run_concurrent(&task);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   232
  return !task.failed();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   233
}