src/hotspot/share/gc/z/zRelocate.cpp
author pliden
Mon, 18 Mar 2019 11:50:39 +0100
changeset 54161 349843ebb209
parent 53843 b38d76fc4835
child 54162 f344a0c6e19e
permissions -rw-r--r--
8220586: ZGC: Move relocation logic from ZPage to ZRelocate Reviewed-by: stefank, eosterlund
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     1
/*
53842
c459deff5939 8219331: ZGC: Unify TLAB retire/remap handling
pliden
parents: 51818
diff changeset
     2
 * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     4
 *
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     5
 * This code is free software; you can redistribute it and/or modify it
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     6
 * under the terms of the GNU General Public License version 2 only, as
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     7
 * published by the Free Software Foundation.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     8
 *
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
     9
 * This code is distributed in the hope that it will be useful, but WITHOUT
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    12
 * version 2 for more details (a copy is included in the LICENSE file that
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    13
 * accompanied this code).
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    14
 *
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    15
 * You should have received a copy of the GNU General Public License version
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    16
 * 2 along with this work; if not, write to the Free Software Foundation,
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    18
 *
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    20
 * or visit www.oracle.com if you need additional information or have any
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    21
 * questions.
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    22
 */
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    23
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    24
#include "precompiled.hpp"
51818
75e4ce0fa1ba 8210883: ZGC: Parallel retire/resize/remap of TLABs
pliden
parents: 51803
diff changeset
    25
#include "gc/z/zAddress.inline.hpp"
75e4ce0fa1ba 8210883: ZGC: Parallel retire/resize/remap of TLABs
pliden
parents: 51803
diff changeset
    26
#include "gc/z/zBarrier.inline.hpp"
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    27
#include "gc/z/zHeap.hpp"
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    28
#include "gc/z/zOopClosures.inline.hpp"
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    29
#include "gc/z/zPage.hpp"
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    30
#include "gc/z/zRelocate.hpp"
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    31
#include "gc/z/zRelocationSet.inline.hpp"
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    32
#include "gc/z/zRootsIterator.hpp"
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    33
#include "gc/z/zStat.hpp"
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    34
#include "gc/z/zTask.hpp"
53842
c459deff5939 8219331: ZGC: Unify TLAB retire/remap handling
pliden
parents: 51818
diff changeset
    35
#include "gc/z/zThreadLocalAllocBuffer.hpp"
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    36
#include "gc/z/zWorkers.hpp"
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    37
#include "logging/log.hpp"
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    38
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    39
static const ZStatCounter ZCounterRelocationContention("Contention", "Relocation Contention", ZStatUnitOpsPerSecond);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    40
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    41
ZRelocate::ZRelocate(ZWorkers* workers) :
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    42
    _workers(workers) {}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    43
51803
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    44
class ZRelocateRootsIteratorClosure : public ZRootsIteratorClosure {
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    45
public:
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    46
  virtual void do_thread(Thread* thread) {
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    47
    // Update thread local address bad mask
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    48
    ZThreadLocalData::set_address_bad_mask(thread, ZAddressBadMask);
51818
75e4ce0fa1ba 8210883: ZGC: Parallel retire/resize/remap of TLABs
pliden
parents: 51803
diff changeset
    49
75e4ce0fa1ba 8210883: ZGC: Parallel retire/resize/remap of TLABs
pliden
parents: 51803
diff changeset
    50
    // Remap TLAB
53842
c459deff5939 8219331: ZGC: Unify TLAB retire/remap handling
pliden
parents: 51818
diff changeset
    51
    ZThreadLocalAllocBuffer::remap(thread);
51803
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    52
  }
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    53
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    54
  virtual void do_oop(oop* p) {
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    55
    ZBarrier::relocate_barrier_on_root_oop_field(p);
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    56
  }
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    57
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    58
  virtual void do_oop(narrowOop* p) {
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    59
    ShouldNotReachHere();
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    60
  }
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    61
};
a16777c0a6c5 8210881: ZGC: Introduce ZRootsIteratorClosure
pliden
parents: 50525
diff changeset
    62
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    63
class ZRelocateRootsTask : public ZTask {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    64
private:
51818
75e4ce0fa1ba 8210883: ZGC: Parallel retire/resize/remap of TLABs
pliden
parents: 51803
diff changeset
    65
  ZRootsIterator                _roots;
75e4ce0fa1ba 8210883: ZGC: Parallel retire/resize/remap of TLABs
pliden
parents: 51803
diff changeset
    66
  ZRelocateRootsIteratorClosure _cl;
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    67
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    68
public:
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    69
  ZRelocateRootsTask() :
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    70
      ZTask("ZRelocateRootsTask"),
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    71
      _roots() {}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    72
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    73
  virtual void work() {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    74
    // During relocation we need to visit the JVMTI
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    75
    // export weak roots to rehash the JVMTI tag map
51818
75e4ce0fa1ba 8210883: ZGC: Parallel retire/resize/remap of TLABs
pliden
parents: 51803
diff changeset
    76
    _roots.oops_do(&_cl, true /* visit_jvmti_weak_export */);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    77
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    78
};
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    79
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    80
void ZRelocate::start() {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    81
  ZRelocateRootsTask task;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    82
  _workers->run_parallel(&task);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    83
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
    84
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    85
uintptr_t ZRelocate::relocate_object_inner(ZPage* page, uintptr_t from_index, uintptr_t from_offset) const {
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    86
  ZForwardingTableCursor cursor;
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    87
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    88
  // Lookup address in forwarding table
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    89
  const ZForwardingTableEntry entry = page->find_forwarding(from_index, &cursor);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    90
  if (entry.from_index() == from_index) {
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    91
    // Already relocated, return new address
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    92
    return entry.to_offset();
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    93
  }
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    94
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    95
  assert(ZHeap::heap()->is_object_live(ZAddress::good(from_offset)), "Should be live");
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    96
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    97
  if (page->is_pinned()) {
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    98
    // In-place forward
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
    99
    return page->insert_forwarding(from_index, from_offset, &cursor);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   100
  }
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   101
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   102
  // Allocate object
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   103
  const uintptr_t from_good = ZAddress::good(from_offset);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   104
  const size_t size = ZUtils::object_size(from_good);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   105
  const uintptr_t to_good = ZHeap::heap()->alloc_object_for_relocation(size);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   106
  if (to_good == 0) {
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   107
    // Failed, in-place forward
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   108
    return page->insert_forwarding(from_index, from_offset, &cursor);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   109
  }
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   110
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   111
  // Copy object
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   112
  ZUtils::object_copy(from_good, to_good, size);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   113
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   114
  // Update forwarding table
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   115
  const uintptr_t to_offset = ZAddress::offset(to_good);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   116
  const uintptr_t to_offset_final = page->insert_forwarding(from_index, to_offset, &cursor);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   117
  if (to_offset_final == to_offset) {
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   118
    // Relocation succeeded
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   119
    return to_offset;
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   120
  }
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   121
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   122
  // Relocation contention
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   123
  ZStatInc(ZCounterRelocationContention);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   124
  log_trace(gc)("Relocation contention, thread: " PTR_FORMAT " (%s), page: " PTR_FORMAT
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   125
                ", entry: " SIZE_FORMAT ", oop: " PTR_FORMAT ", size: " SIZE_FORMAT,
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   126
                ZThread::id(), ZThread::name(), p2i(this), cursor, from_good, size);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   127
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   128
  // Try undo allocation
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   129
  ZHeap::heap()->undo_alloc_object_for_relocation(to_good, size);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   130
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   131
  return to_offset_final;
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   132
}
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   133
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   134
uintptr_t ZRelocate::relocate_object(ZPage* page, uintptr_t from_addr) const {
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   135
  assert(ZHeap::heap()->is_relocating(from_addr), "Should be relocating");
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   136
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   137
  const uintptr_t from_offset = ZAddress::offset(from_addr);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   138
  const uintptr_t from_index = (from_offset - page->start()) >> page->object_alignment_shift();
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   139
  const uintptr_t to_offset = relocate_object_inner(page, from_index, from_offset);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   140
  if (from_offset == to_offset) {
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   141
    // In-place forwarding, pin page
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   142
    page->set_pinned();
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   143
  }
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   144
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   145
  return ZAddress::good(to_offset);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   146
}
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   147
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   148
uintptr_t ZRelocate::forward_object(ZPage* page, uintptr_t from_addr) const {
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   149
  assert(ZHeap::heap()->is_relocating(from_addr), "Should be relocated");
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   150
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   151
  // Lookup address in forwarding table
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   152
  const uintptr_t from_offset = ZAddress::offset(from_addr);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   153
  const uintptr_t from_index = (from_offset - page->start()) >> page->object_alignment_shift();
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   154
  const ZForwardingTableEntry entry = page->find_forwarding(from_index);
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   155
  assert(entry.from_index() == from_index, "Should be forwarded");
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   156
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   157
  return ZAddress::good(entry.to_offset());
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   158
}
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   159
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   160
class ZRelocateObjectClosure : public ObjectClosure {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   161
private:
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   162
  ZRelocate* const _relocate;
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   163
  ZPage* const     _page;
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   164
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   165
public:
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   166
  ZRelocateObjectClosure(ZRelocate* relocate, ZPage* page) :
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   167
      _relocate(relocate),
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   168
      _page(page) {}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   169
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   170
  virtual void do_object(oop o) {
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   171
    _relocate->relocate_object(_page, ZOop::to_address(o));
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   172
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   173
};
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   174
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   175
bool ZRelocate::work(ZRelocationSetParallelIterator* iter) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   176
  bool success = true;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   177
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   178
  // Relocate pages in the relocation set
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   179
  for (ZPage* page; iter->next(&page);) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   180
    // Relocate objects in page
54161
349843ebb209 8220586: ZGC: Move relocation logic from ZPage to ZRelocate
pliden
parents: 53843
diff changeset
   181
    ZRelocateObjectClosure cl(this, page);
50525
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   182
    page->object_iterate(&cl);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   183
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   184
    if (ZVerifyForwarding) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   185
      page->verify_forwarding();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   186
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   187
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   188
    if (page->is_pinned()) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   189
      // Relocation failed, page is now pinned
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   190
      success = false;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   191
    } else {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   192
      // Relocation succeeded, release page
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   193
      ZHeap::heap()->release_page(page, true /* reclaimed */);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   194
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   195
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   196
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   197
  return success;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   198
}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   199
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   200
class ZRelocateTask : public ZTask {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   201
private:
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   202
  ZRelocate* const               _relocate;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   203
  ZRelocationSetParallelIterator _iter;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   204
  bool                           _failed;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   205
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   206
public:
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   207
  ZRelocateTask(ZRelocate* relocate, ZRelocationSet* relocation_set) :
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   208
      ZTask("ZRelocateTask"),
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   209
      _relocate(relocate),
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   210
      _iter(relocation_set),
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   211
      _failed(false) {}
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   212
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   213
  virtual void work() {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   214
    if (!_relocate->work(&_iter)) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   215
      _failed = true;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   216
    }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   217
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   218
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   219
  bool failed() const {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   220
    return _failed;
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   221
  }
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   222
};
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   223
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   224
bool ZRelocate::relocate(ZRelocationSet* relocation_set) {
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   225
  ZRelocateTask task(this, relocation_set);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   226
  _workers->run_concurrent(&task);
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   227
  return !task.failed();
767cdb97f103 8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff changeset
   228
}