8214527: ZGC for Aarch64
Summary: Implement ZGC for AArch64
Reviewed-by: aph, pliden, eosterlund, rkennke, shade, njian, adinn
--- a/make/autoconf/hotspot.m4 Thu Jun 13 12:27:29 2019 +0200
+++ b/make/autoconf/hotspot.m4 Thu Jun 13 15:24:34 2019 +0100
@@ -350,7 +350,8 @@
# Only enable ZGC on supported platforms
AC_MSG_CHECKING([if zgc can be built])
- if test "x$OPENJDK_TARGET_OS" = "xlinux" && test "x$OPENJDK_TARGET_CPU" = "xx86_64"; then
+ if (test "x$OPENJDK_TARGET_OS" = "xlinux" && test "x$OPENJDK_TARGET_CPU" = "xx86_64") || \
+ (test "x$OPENJDK_TARGET_OS" = "xlinux" && test "x$OPENJDK_TARGET_CPU" = "xaarch64"); then
AC_MSG_RESULT([yes])
else
DISABLED_JVM_FEATURES="$DISABLED_JVM_FEATURES zgc"
--- a/src/hotspot/cpu/aarch64/aarch64.ad Thu Jun 13 12:27:29 2019 +0200
+++ b/src/hotspot/cpu/aarch64/aarch64.ad Thu Jun 13 15:24:34 2019 +0100
@@ -957,6 +957,146 @@
V3, V3_H
);
+// Class for 128 bit register v4
+reg_class v4_reg(
+ V4, V4_H
+);
+
+// Class for 128 bit register v5
+reg_class v5_reg(
+ V5, V5_H
+);
+
+// Class for 128 bit register v6
+reg_class v6_reg(
+ V6, V6_H
+);
+
+// Class for 128 bit register v7
+reg_class v7_reg(
+ V7, V7_H
+);
+
+// Class for 128 bit register v8
+reg_class v8_reg(
+ V8, V8_H
+);
+
+// Class for 128 bit register v9
+reg_class v9_reg(
+ V9, V9_H
+);
+
+// Class for 128 bit register v10
+reg_class v10_reg(
+ V10, V10_H
+);
+
+// Class for 128 bit register v11
+reg_class v11_reg(
+ V11, V11_H
+);
+
+// Class for 128 bit register v12
+reg_class v12_reg(
+ V12, V12_H
+);
+
+// Class for 128 bit register v13
+reg_class v13_reg(
+ V13, V13_H
+);
+
+// Class for 128 bit register v14
+reg_class v14_reg(
+ V14, V14_H
+);
+
+// Class for 128 bit register v15
+reg_class v15_reg(
+ V15, V15_H
+);
+
+// Class for 128 bit register v16
+reg_class v16_reg(
+ V16, V16_H
+);
+
+// Class for 128 bit register v17
+reg_class v17_reg(
+ V17, V17_H
+);
+
+// Class for 128 bit register v18
+reg_class v18_reg(
+ V18, V18_H
+);
+
+// Class for 128 bit register v19
+reg_class v19_reg(
+ V19, V19_H
+);
+
+// Class for 128 bit register v20
+reg_class v20_reg(
+ V20, V20_H
+);
+
+// Class for 128 bit register v21
+reg_class v21_reg(
+ V21, V21_H
+);
+
+// Class for 128 bit register v22
+reg_class v22_reg(
+ V22, V22_H
+);
+
+// Class for 128 bit register v23
+reg_class v23_reg(
+ V23, V23_H
+);
+
+// Class for 128 bit register v24
+reg_class v24_reg(
+ V24, V24_H
+);
+
+// Class for 128 bit register v25
+reg_class v25_reg(
+ V25, V25_H
+);
+
+// Class for 128 bit register v26
+reg_class v26_reg(
+ V26, V26_H
+);
+
+// Class for 128 bit register v27
+reg_class v27_reg(
+ V27, V27_H
+);
+
+// Class for 128 bit register v28
+reg_class v28_reg(
+ V28, V28_H
+);
+
+// Class for 128 bit register v29
+reg_class v29_reg(
+ V29, V29_H
+);
+
+// Class for 128 bit register v30
+reg_class v30_reg(
+ V30, V30_H
+);
+
+// Class for 128 bit register v31
+reg_class v31_reg(
+ V31, V31_H
+);
+
// Singleton class for condition codes
reg_class int_flags(RFLAGS);
@@ -4774,6 +4914,258 @@
interface(REG_INTER);
%}
+operand vRegD_V4()
+%{
+ constraint(ALLOC_IN_RC(v4_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V5()
+%{
+ constraint(ALLOC_IN_RC(v5_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V6()
+%{
+ constraint(ALLOC_IN_RC(v6_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V7()
+%{
+ constraint(ALLOC_IN_RC(v7_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V8()
+%{
+ constraint(ALLOC_IN_RC(v8_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V9()
+%{
+ constraint(ALLOC_IN_RC(v9_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V10()
+%{
+ constraint(ALLOC_IN_RC(v10_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V11()
+%{
+ constraint(ALLOC_IN_RC(v11_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V12()
+%{
+ constraint(ALLOC_IN_RC(v12_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V13()
+%{
+ constraint(ALLOC_IN_RC(v13_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V14()
+%{
+ constraint(ALLOC_IN_RC(v14_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V15()
+%{
+ constraint(ALLOC_IN_RC(v15_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V16()
+%{
+ constraint(ALLOC_IN_RC(v16_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V17()
+%{
+ constraint(ALLOC_IN_RC(v17_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V18()
+%{
+ constraint(ALLOC_IN_RC(v18_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V19()
+%{
+ constraint(ALLOC_IN_RC(v19_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V20()
+%{
+ constraint(ALLOC_IN_RC(v20_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V21()
+%{
+ constraint(ALLOC_IN_RC(v21_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V22()
+%{
+ constraint(ALLOC_IN_RC(v22_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V23()
+%{
+ constraint(ALLOC_IN_RC(v23_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V24()
+%{
+ constraint(ALLOC_IN_RC(v24_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V25()
+%{
+ constraint(ALLOC_IN_RC(v25_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V26()
+%{
+ constraint(ALLOC_IN_RC(v26_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V27()
+%{
+ constraint(ALLOC_IN_RC(v27_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V28()
+%{
+ constraint(ALLOC_IN_RC(v28_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V29()
+%{
+ constraint(ALLOC_IN_RC(v29_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V30()
+%{
+ constraint(ALLOC_IN_RC(v30_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
+operand vRegD_V31()
+%{
+ constraint(ALLOC_IN_RC(v31_reg));
+ match(RegD);
+ op_cost(0);
+ format %{ %}
+ interface(REG_INTER);
+%}
+
// Flags register, used as output of signed compare instructions
// note that on AArch64 we also use this register as the output for
--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp Thu Jun 13 12:27:29 2019 +0200
+++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp Thu Jun 13 15:24:34 2019 +0100
@@ -1015,7 +1015,11 @@
if (UseCompressedOops && !wide) {
__ decode_heap_oop(dest->as_register());
}
- __ verify_oop(dest->as_register());
+
+ if (!UseZGC) {
+ // Load barrier has not yet been applied, so ZGC can't verify the oop here
+ __ verify_oop(dest->as_register());
+ }
} else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
if (UseCompressedClassPointers) {
__ decode_klass_not_null(dest->as_register());
@@ -2869,7 +2873,11 @@
void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
- assert(patch_code == lir_patch_none, "Patch code not supported");
+ if (patch_code != lir_patch_none) {
+ deoptimize_trap(info);
+ return;
+ }
+
__ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp Thu Jun 13 15:24:34 2019 +0100
@@ -0,0 +1,407 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "asm/macroAssembler.inline.hpp"
+#include "code/codeBlob.hpp"
+#include "gc/z/zBarrier.inline.hpp"
+#include "gc/z/zBarrierSet.hpp"
+#include "gc/z/zBarrierSetAssembler.hpp"
+#include "gc/z/zBarrierSetRuntime.hpp"
+#include "memory/resourceArea.hpp"
+#ifdef COMPILER1
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "gc/z/c1/zBarrierSetC1.hpp"
+#endif // COMPILER1
+
+#include "gc/z/zThreadLocalData.hpp"
+
+ZBarrierSetAssembler::ZBarrierSetAssembler() :
+ _load_barrier_slow_stub(),
+ _load_barrier_weak_slow_stub() {}
+
+#ifdef PRODUCT
+#define BLOCK_COMMENT(str) /* nothing */
+#else
+#define BLOCK_COMMENT(str) __ block_comment(str)
+#endif
+
+#undef __
+#define __ masm->
+
+void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
+ DecoratorSet decorators,
+ BasicType type,
+ Register dst,
+ Address src,
+ Register tmp1,
+ Register tmp_thread) {
+ if (!ZBarrierSet::barrier_needed(decorators, type)) {
+ // Barrier not needed
+ BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+ return;
+ }
+
+ // rscratch1 can be passed as src or dst, so don't use it.
+ RegSet savedRegs = RegSet::of(rscratch2, rheapbase);
+
+ Label done;
+ assert_different_registers(rheapbase, rscratch2, dst);
+ assert_different_registers(rheapbase, rscratch2, src.base());
+
+ __ push(savedRegs, sp);
+
+ // Load bad mask into scratch register.
+ __ ldr(rheapbase, address_bad_mask_from_thread(rthread));
+ __ lea(rscratch2, src);
+ __ ldr(dst, src);
+
+ // Test reference against bad mask. If mask bad, then we need to fix it up.
+ __ tst(dst, rheapbase);
+ __ br(Assembler::EQ, done);
+
+ __ enter();
+
+ __ push(RegSet::range(r0,r28) - RegSet::of(dst), sp);
+
+ if (c_rarg0 != dst) {
+ __ mov(c_rarg0, dst);
+ }
+ __ mov(c_rarg1, rscratch2);
+
+ int step = 4 * wordSize;
+ __ mov(rscratch1, -step);
+ __ sub(sp, sp, step);
+
+ for (int i = 28; i >= 4; i -= 4) {
+ __ st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
+ as_FloatRegister(i+3), __ T1D, Address(__ post(sp, rscratch1)));
+ }
+
+ __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
+
+ for (int i = 0; i <= 28; i += 4) {
+ __ ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
+ as_FloatRegister(i+3), __ T1D, Address(__ post(sp, step)));
+ }
+
+ // Make sure dst has the return value.
+ if (dst != r0) {
+ __ mov(dst, r0);
+ }
+
+ __ pop(RegSet::range(r0,r28) - RegSet::of(dst), sp);
+ __ leave();
+
+ __ bind(done);
+
+ // Restore tmps
+ __ pop(savedRegs, sp);
+}
+
+#ifdef ASSERT
+
+void ZBarrierSetAssembler::store_at(MacroAssembler* masm,
+ DecoratorSet decorators,
+ BasicType type,
+ Address dst,
+ Register val,
+ Register tmp1,
+ Register tmp2) {
+ // Verify value
+ if (type == T_OBJECT || type == T_ARRAY) {
+ // Note that src could be noreg, which means we
+ // are storing null and can skip verification.
+ if (val != noreg) {
+ Label done;
+
+ // tmp1 and tmp2 are often set to noreg.
+ RegSet savedRegs = RegSet::of(rscratch1);
+ __ push(savedRegs, sp);
+
+ __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
+ __ tst(val, rscratch1);
+ __ br(Assembler::EQ, done);
+ __ stop("Verify oop store failed");
+ __ should_not_reach_here();
+ __ bind(done);
+ __ pop(savedRegs, sp);
+ }
+ }
+
+ // Store value
+ BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
+}
+
+#endif // ASSERT
+
+void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
+ DecoratorSet decorators,
+ bool is_oop,
+ Register src,
+ Register dst,
+ Register count,
+ RegSet saved_regs) {
+ if (!is_oop) {
+ // Barrier not needed
+ return;
+ }
+
+ BLOCK_COMMENT("ZBarrierSetAssembler::arraycopy_prologue {");
+
+ assert_different_registers(src, count, rscratch1);
+
+ __ pusha();
+
+ if (count == c_rarg0) {
+ if (src == c_rarg1) {
+ // exactly backwards!!
+ __ mov(rscratch1, c_rarg0);
+ __ mov(c_rarg0, c_rarg1);
+ __ mov(c_rarg1, rscratch1);
+ } else {
+ __ mov(c_rarg1, count);
+ __ mov(c_rarg0, src);
+ }
+ } else {
+ __ mov(c_rarg0, src);
+ __ mov(c_rarg1, count);
+ }
+
+ __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_array_addr(), 2);
+
+ __ popa();
+ BLOCK_COMMENT("} ZBarrierSetAssembler::arraycopy_prologue");
+}
+
+void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
+ Register jni_env,
+ Register robj,
+ Register tmp,
+ Label& slowpath) {
+ BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_jobject_in_native {");
+
+ assert_different_registers(jni_env, robj, tmp);
+
+ // Resolve jobject
+ BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, robj, tmp, slowpath);
+
+ // The Address offset is too large to direct load - -784. Our range is +127, -128.
+ __ mov(tmp, (long int)(in_bytes(ZThreadLocalData::address_bad_mask_offset()) -
+ in_bytes(JavaThread::jni_environment_offset())));
+ // Load address bad mask
+ __ add(tmp, jni_env, tmp);
+ __ ldr(tmp, Address(tmp));
+
+ // Check address bad mask
+ __ tst(robj, tmp);
+ __ br(Assembler::NE, slowpath);
+
+ BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native");
+}
+
+#ifdef COMPILER1
+
+#undef __
+#define __ ce->masm()->
+
+void ZBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
+ LIR_Opr ref) const {
+ assert_different_registers(rheapbase, rthread, ref->as_register());
+
+ __ ldr(rheapbase, address_bad_mask_from_thread(rthread));
+ __ tst(ref->as_register(), rheapbase);
+}
+
+void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
+ ZLoadBarrierStubC1* stub) const {
+ // Stub entry
+ __ bind(*stub->entry());
+
+ Register ref = stub->ref()->as_register();
+ Register ref_addr = noreg;
+ Register tmp = noreg;
+
+ if (stub->tmp()->is_valid()) {
+ // Load address into tmp register
+ ce->leal(stub->ref_addr(), stub->tmp());
+ ref_addr = tmp = stub->tmp()->as_pointer_register();
+ } else {
+ // Address already in register
+ ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register();
+ }
+
+ assert_different_registers(ref, ref_addr, noreg);
+
+ // Save r0 unless it is the result or tmp register
+ // Set up SP to accomodate parameters and maybe r0..
+ if (ref != r0 && tmp != r0) {
+ __ sub(sp, sp, 32);
+ __ str(r0, Address(sp, 16));
+ } else {
+ __ sub(sp, sp, 16);
+ }
+
+ // Setup arguments and call runtime stub
+ ce->store_parameter(ref_addr, 1);
+ ce->store_parameter(ref, 0);
+
+ __ far_call(stub->runtime_stub());
+
+ // Verify result
+ __ verify_oop(r0, "Bad oop");
+
+ // Move result into place
+ if (ref != r0) {
+ __ mov(ref, r0);
+ }
+
+ // Restore r0 unless it is the result or tmp register
+ if (ref != r0 && tmp != r0) {
+ __ ldr(r0, Address(sp, 16));
+ __ add(sp, sp, 32);
+ } else {
+ __ add(sp, sp, 16);
+ }
+
+ // Stub exit
+ __ b(*stub->continuation());
+}
+
+#undef __
+#define __ sasm->
+
+void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
+ DecoratorSet decorators) const {
+ __ prologue("zgc_load_barrier stub", false);
+
+ // We don't use push/pop_clobbered_registers() - we need to pull out the result from r0.
+ for (int i = 0; i < 32; i +=2) {
+ __ stpd(as_FloatRegister(i), as_FloatRegister(i+1), Address(__ pre(sp,-16)));
+ }
+
+ RegSet saveRegs = RegSet::range(r0,r28) - RegSet::of(r0);
+ __ push(saveRegs, sp);
+
+ // Setup arguments
+ __ load_parameter(0, c_rarg0);
+ __ load_parameter(1, c_rarg1);
+
+ __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
+
+ __ pop(saveRegs, sp);
+
+ for (int i = 30; i >0; i -=2) {
+ __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1), Address(__ post(sp, 16)));
+ }
+
+ __ epilogue();
+}
+#endif // COMPILER1
+
+#undef __
+#define __ cgen->assembler()->
+
+// Generates a register specific stub for calling
+// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or
+// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded().
+//
+// The raddr register serves as both input and output for this stub. When the stub is
+// called the raddr register contains the object field address (oop*) where the bad oop
+// was loaded from, which caused the slow path to be taken. On return from the stub the
+// raddr register contains the good/healed oop returned from
+// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or
+// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded().
+static address generate_load_barrier_stub(StubCodeGenerator* cgen, Register raddr, DecoratorSet decorators) {
+ // Don't generate stub for invalid registers
+ if (raddr == zr || raddr == r29 || raddr == r30) {
+ return NULL;
+ }
+
+ // Create stub name
+ char name[64];
+ const bool weak = (decorators & ON_WEAK_OOP_REF) != 0;
+ os::snprintf(name, sizeof(name), "zgc_load_barrier%s_stub_%s", weak ? "_weak" : "", raddr->name());
+
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(cgen, "StubRoutines", os::strdup(name, mtCode));
+ address start = __ pc();
+
+ // Save live registers
+ RegSet savedRegs = RegSet::range(r0,r18) - RegSet::of(raddr);
+
+ __ enter();
+ __ push(savedRegs, sp);
+
+ // Setup arguments
+ if (raddr != c_rarg1) {
+ __ mov(c_rarg1, raddr);
+ }
+
+ __ ldr(c_rarg0, Address(raddr));
+
+ // Call barrier function
+ __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1);
+
+ // Move result returned in r0 to raddr, if needed
+ if (raddr != r0) {
+ __ mov(raddr, r0);
+ }
+
+ __ pop(savedRegs, sp);
+ __ leave();
+ __ ret(lr);
+
+ return start;
+}
+
+#undef __
+
+static void barrier_stubs_init_inner(const char* label, const DecoratorSet decorators, address* stub) {
+ const int nregs = 28; // Exclude FP, XZR, SP from calculation.
+ const int code_size = nregs * 254; // Rough estimate of code size
+
+ ResourceMark rm;
+
+ CodeBuffer buf(BufferBlob::create(label, code_size));
+ StubCodeGenerator cgen(&buf);
+
+ for (int i = 0; i < nregs; i++) {
+ const Register reg = as_Register(i);
+ stub[i] = generate_load_barrier_stub(&cgen, reg, decorators);
+ }
+}
+
+void ZBarrierSetAssembler::barrier_stubs_init() {
+ barrier_stubs_init_inner("zgc_load_barrier_stubs", ON_STRONG_OOP_REF, _load_barrier_slow_stub);
+ barrier_stubs_init_inner("zgc_load_barrier_weak_stubs", ON_WEAK_OOP_REF, _load_barrier_weak_slow_stub);
+}
+
+address ZBarrierSetAssembler::load_barrier_slow_stub(Register reg) {
+ return _load_barrier_slow_stub[reg->encoding()];
+}
+
+address ZBarrierSetAssembler::load_barrier_weak_slow_stub(Register reg) {
+ return _load_barrier_weak_slow_stub[reg->encoding()];
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp Thu Jun 13 15:24:34 2019 +0100
@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP
+#define CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP
+
+#ifdef COMPILER1
+class LIR_Assembler;
+class LIR_OprDesc;
+typedef LIR_OprDesc* LIR_Opr;
+class StubAssembler;
+class ZLoadBarrierStubC1;
+#endif // COMPILER1
+
+class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase {
+private:
+ address _load_barrier_slow_stub[RegisterImpl::number_of_registers];
+ address _load_barrier_weak_slow_stub[RegisterImpl::number_of_registers];
+
+public:
+ ZBarrierSetAssembler();
+
+ virtual void load_at(MacroAssembler* masm,
+ DecoratorSet decorators,
+ BasicType type,
+ Register dst,
+ Address src,
+ Register tmp1,
+ Register tmp_thread);
+
+#ifdef ASSERT
+ virtual void store_at(MacroAssembler* masm,
+ DecoratorSet decorators,
+ BasicType type,
+ Address dst,
+ Register val,
+ Register tmp1,
+ Register tmp2);
+#endif // ASSERT
+
+ virtual void arraycopy_prologue(MacroAssembler* masm,
+ DecoratorSet decorators,
+ bool is_oop,
+ Register src,
+ Register dst,
+ Register count,
+ RegSet saved_regs);
+
+ virtual void try_resolve_jobject_in_native(MacroAssembler* masm,
+ Register jni_env,
+ Register robj,
+ Register tmp,
+ Label& slowpath);
+
+#ifdef COMPILER1
+ void generate_c1_load_barrier_test(LIR_Assembler* ce,
+ LIR_Opr ref) const;
+
+ void generate_c1_load_barrier_stub(LIR_Assembler* ce,
+ ZLoadBarrierStubC1* stub) const;
+
+ void generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
+ DecoratorSet decorators) const;
+#endif // COMPILER1
+
+ virtual void barrier_stubs_init();
+
+ address load_barrier_slow_stub(Register reg);
+ address load_barrier_weak_slow_stub(Register reg);
+};
+
+#endif // CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad Thu Jun 13 15:24:34 2019 +0100
@@ -0,0 +1,174 @@
+//
+// Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+
+source_hpp %{
+
+#include "gc/z/c2/zBarrierSetC2.hpp"
+
+%}
+
+source %{
+
+#include "gc/z/zBarrierSetAssembler.hpp"
+
+static void z_load_barrier_slow_reg(MacroAssembler& _masm, Register dst,
+ Register base, int index, int scale,
+ int disp, bool weak) {
+ const address stub = weak ? ZBarrierSet::assembler()->load_barrier_weak_slow_stub(dst)
+ : ZBarrierSet::assembler()->load_barrier_slow_stub(dst);
+
+ if (index == -1) {
+ if (disp != 0) {
+ __ lea(dst, Address(base, disp));
+ } else {
+ __ mov(dst, base);
+ }
+ } else {
+ Register index_reg = as_Register(index);
+ if (disp == 0) {
+ __ lea(dst, Address(base, index_reg, Address::lsl(scale)));
+ } else {
+ __ lea(dst, Address(base, disp));
+ __ lea(dst, Address(dst, index_reg, Address::lsl(scale)));
+ }
+ }
+
+ __ far_call(RuntimeAddress(stub));
+}
+
+%}
+
+//
+// Execute ZGC load barrier (strong) slow path
+//
+instruct loadBarrierSlowReg(iRegP dst, memory mem, rFlagsReg cr,
+ vRegD_V0 v0, vRegD_V1 v1, vRegD_V2 v2, vRegD_V3 v3, vRegD_V4 v4,
+ vRegD_V5 v5, vRegD_V6 v6, vRegD_V7 v7, vRegD_V8 v8, vRegD_V9 v9,
+ vRegD_V10 v10, vRegD_V11 v11, vRegD_V12 v12, vRegD_V13 v13, vRegD_V14 v14,
+ vRegD_V15 v15, vRegD_V16 v16, vRegD_V17 v17, vRegD_V18 v18, vRegD_V19 v19,
+ vRegD_V20 v20, vRegD_V21 v21, vRegD_V22 v22, vRegD_V23 v23, vRegD_V24 v24,
+ vRegD_V25 v25, vRegD_V26 v26, vRegD_V27 v27, vRegD_V28 v28, vRegD_V29 v29,
+ vRegD_V30 v30, vRegD_V31 v31) %{
+ match(Set dst (LoadBarrierSlowReg mem));
+ predicate(!n->as_LoadBarrierSlowReg()->is_weak());
+
+ effect(DEF dst, KILL cr,
+ KILL v0, KILL v1, KILL v2, KILL v3, KILL v4, KILL v5, KILL v6, KILL v7,
+ KILL v8, KILL v9, KILL v10, KILL v11, KILL v12, KILL v13, KILL v14,
+ KILL v15, KILL v16, KILL v17, KILL v18, KILL v19, KILL v20, KILL v21,
+ KILL v22, KILL v23, KILL v24, KILL v25, KILL v26, KILL v27, KILL v28,
+ KILL v29, KILL v30, KILL v31);
+
+ format %{"LoadBarrierSlowReg $dst, $mem" %}
+ ins_encode %{
+ z_load_barrier_slow_reg(_masm, $dst$$Register, $mem$$base$$Register,
+ $mem$$index, $mem$$scale, $mem$$disp, false);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+//
+// Execute ZGC load barrier (weak) slow path
+//
+instruct loadBarrierWeakSlowReg(iRegP dst, memory mem, rFlagsReg cr,
+ vRegD_V0 v0, vRegD_V1 v1, vRegD_V2 v2, vRegD_V3 v3, vRegD_V4 v4,
+ vRegD_V5 v5, vRegD_V6 v6, vRegD_V7 v7, vRegD_V8 v8, vRegD_V9 v9,
+ vRegD_V10 v10, vRegD_V11 v11, vRegD_V12 v12, vRegD_V13 v13, vRegD_V14 v14,
+ vRegD_V15 v15, vRegD_V16 v16, vRegD_V17 v17, vRegD_V18 v18, vRegD_V19 v19,
+ vRegD_V20 v20, vRegD_V21 v21, vRegD_V22 v22, vRegD_V23 v23, vRegD_V24 v24,
+ vRegD_V25 v25, vRegD_V26 v26, vRegD_V27 v27, vRegD_V28 v28, vRegD_V29 v29,
+ vRegD_V30 v30, vRegD_V31 v31) %{
+ match(Set dst (LoadBarrierSlowReg mem));
+ predicate(n->as_LoadBarrierSlowReg()->is_weak());
+
+ effect(DEF dst, KILL cr,
+ KILL v0, KILL v1, KILL v2, KILL v3, KILL v4, KILL v5, KILL v6, KILL v7,
+ KILL v8, KILL v9, KILL v10, KILL v11, KILL v12, KILL v13, KILL v14,
+ KILL v15, KILL v16, KILL v17, KILL v18, KILL v19, KILL v20, KILL v21,
+ KILL v22, KILL v23, KILL v24, KILL v25, KILL v26, KILL v27, KILL v28,
+ KILL v29, KILL v30, KILL v31);
+
+ format %{"LoadBarrierWeakSlowReg $dst, $mem" %}
+ ins_encode %{
+ z_load_barrier_slow_reg(_masm, $dst$$Register, $mem$$base$$Register,
+ $mem$$index, $mem$$scale, $mem$$disp, true);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+
+// Specialized versions of compareAndExchangeP that adds a keepalive that is consumed
+// but doesn't affect output.
+
+instruct z_compareAndExchangeP(iRegPNoSp res, indirect mem,
+ iRegP oldval, iRegP newval, iRegP keepalive,
+ rFlagsReg cr) %{
+ match(Set res (ZCompareAndExchangeP (Binary mem keepalive) (Binary oldval newval)));
+ ins_cost(2 * VOLATILE_REF_COST);
+ effect(TEMP_DEF res, KILL cr);
+ format %{
+ "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
+ %}
+ ins_encode %{
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
+ Assembler::xword, /*acquire*/ false, /*release*/ true,
+ /*weak*/ false, $res$$Register);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct z_compareAndSwapP(iRegINoSp res,
+ indirect mem,
+ iRegP oldval, iRegP newval, iRegP keepalive,
+ rFlagsReg cr) %{
+
+ match(Set res (ZCompareAndSwapP (Binary mem keepalive) (Binary oldval newval)));
+ match(Set res (ZWeakCompareAndSwapP (Binary mem keepalive) (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ effect(KILL cr);
+
+ format %{
+ "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
+ "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
+ %}
+
+ ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
+ aarch64_enc_cset_eq(res));
+
+ ins_pipe(pipe_slow);
+%}
+
+
+instruct z_get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev,
+ iRegP keepalive) %{
+ match(Set prev (ZGetAndSetP mem (Binary newv keepalive)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+ format %{ "atomic_xchg $prev, $newv, [$mem]" %}
+ ins_encode %{
+ __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
+ %}
+ ins_pipe(pipe_serial);
+%}
--- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp Thu Jun 13 12:27:29 2019 +0200
+++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp Thu Jun 13 15:24:34 2019 +0100
@@ -46,6 +46,9 @@
#ifdef COMPILER2
#include "opto/runtime.hpp"
#endif
+#if INCLUDE_ZGC
+#include "gc/z/zThreadLocalData.hpp"
+#endif
#ifdef BUILTIN_SIM
#include "../../../../../../simulator/simulator.hpp"
@@ -580,6 +583,16 @@
// make sure object is 'reasonable'
__ cbz(r0, exit); // if obj is NULL it is OK
+#if INCLUDE_ZGC
+ if (UseZGC) {
+ // Check if mask is good.
+ // verifies that ZAddressBadMask & r0 == 0
+ __ ldr(c_rarg3, Address(rthread, ZThreadLocalData::address_bad_mask_offset()));
+ __ andr(c_rarg2, r0, c_rarg3);
+ __ cbnz(c_rarg2, error);
+ }
+#endif
+
// Check if the oop is in the right area of memory
__ mov(c_rarg3, (intptr_t) Universe::verify_oop_mask());
__ andr(c_rarg2, r0, c_rarg3);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zArguments_linux_aarch64.cpp Thu Jun 13 15:24:34 2019 +0100
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zArguments.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/globals_extension.hpp"
+#include "utilities/debug.hpp"
+
+void ZArguments::initialize_platform() {
+ // Disable class unloading - we don't support concurrent class unloading yet.
+ FLAG_SET_DEFAULT(ClassUnloading, false);
+ FLAG_SET_DEFAULT(ClassUnloadingWithConcurrentMark, false);
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.cpp Thu Jun 13 15:24:34 2019 +0100
@@ -0,0 +1,590 @@
+/*
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zArray.inline.hpp"
+#include "gc/z/zBackingFile_linux_aarch64.hpp"
+#include "gc/z/zBackingPath_linux_aarch64.hpp"
+#include "gc/z/zErrno.hpp"
+#include "gc/z/zGlobals.hpp"
+#include "gc/z/zLargePages.inline.hpp"
+#include "logging/log.hpp"
+#include "runtime/init.hpp"
+#include "runtime/os.hpp"
+#include "utilities/align.hpp"
+#include "utilities/debug.hpp"
+
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/statfs.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+//
+// Support for building on older Linux systems
+//
+
+// System calls
+#ifndef SYS_fallocate
+#define SYS_fallocate 47
+#endif
+#ifndef SYS_memfd_create
+#define SYS_memfd_create 279
+#endif
+
+// memfd_create(2) flags
+#ifndef MFD_CLOEXEC
+#define MFD_CLOEXEC 0x0001U
+#endif
+#ifndef MFD_HUGETLB
+#define MFD_HUGETLB 0x0004U
+#endif
+
+// open(2) flags
+#ifndef O_CLOEXEC
+#define O_CLOEXEC 02000000
+#endif
+#ifndef O_TMPFILE
+#define O_TMPFILE (020000000 | O_DIRECTORY)
+#endif
+
+// fallocate(2) flags
+#ifndef FALLOC_FL_KEEP_SIZE
+#define FALLOC_FL_KEEP_SIZE 0x01
+#endif
+#ifndef FALLOC_FL_PUNCH_HOLE
+#define FALLOC_FL_PUNCH_HOLE 0x02
+#endif
+
+// Filesystem types, see statfs(2)
+#ifndef TMPFS_MAGIC
+#define TMPFS_MAGIC 0x01021994
+#endif
+#ifndef HUGETLBFS_MAGIC
+#define HUGETLBFS_MAGIC 0x958458f6
+#endif
+
+// Filesystem names
+#define ZFILESYSTEM_TMPFS "tmpfs"
+#define ZFILESYSTEM_HUGETLBFS "hugetlbfs"
+
+// Sysfs file for transparent huge page on tmpfs
+#define ZFILENAME_SHMEM_ENABLED "/sys/kernel/mm/transparent_hugepage/shmem_enabled"
+
+// Java heap filename
+#define ZFILENAME_HEAP "java_heap"
+
+// Preferred tmpfs mount points, ordered by priority
+static const char* z_preferred_tmpfs_mountpoints[] = {
+ "/dev/shm",
+ "/run/shm",
+ NULL
+};
+
+// Preferred hugetlbfs mount points, ordered by priority
+static const char* z_preferred_hugetlbfs_mountpoints[] = {
+ "/dev/hugepages",
+ "/hugepages",
+ NULL
+};
+
+static int z_fallocate_hugetlbfs_attempts = 3;
+static bool z_fallocate_supported = true;
+
+static int z_fallocate(int fd, int mode, size_t offset, size_t length) {
+ return syscall(SYS_fallocate, fd, mode, offset, length);
+}
+
+static int z_memfd_create(const char *name, unsigned int flags) {
+ return syscall(SYS_memfd_create, name, flags);
+}
+
+ZBackingFile::ZBackingFile() :
+ _fd(-1),
+ _size(0),
+ _filesystem(0),
+ _block_size(0),
+ _available(0),
+ _initialized(false) {
+
+ // Create backing file
+ _fd = create_fd(ZFILENAME_HEAP);
+ if (_fd == -1) {
+ return;
+ }
+
+ // Get filesystem statistics
+ struct statfs buf;
+ if (fstatfs(_fd, &buf) == -1) {
+ ZErrno err;
+ log_error(gc)("Failed to determine filesystem type for backing file (%s)", err.to_string());
+ return;
+ }
+
+ _filesystem = buf.f_type;
+ _block_size = buf.f_bsize;
+ _available = buf.f_bavail * _block_size;
+
+ // Make sure we're on a supported filesystem
+ if (!is_tmpfs() && !is_hugetlbfs()) {
+ log_error(gc)("Backing file must be located on a %s or a %s filesystem",
+ ZFILESYSTEM_TMPFS, ZFILESYSTEM_HUGETLBFS);
+ return;
+ }
+
+ // Make sure the filesystem type matches requested large page type
+ if (ZLargePages::is_transparent() && !is_tmpfs()) {
+ log_error(gc)("-XX:+UseTransparentHugePages can only be enable when using a %s filesystem",
+ ZFILESYSTEM_TMPFS);
+ return;
+ }
+
+ if (ZLargePages::is_transparent() && !tmpfs_supports_transparent_huge_pages()) {
+ log_error(gc)("-XX:+UseTransparentHugePages on a %s filesystem not supported by kernel",
+ ZFILESYSTEM_TMPFS);
+ return;
+ }
+
+ if (ZLargePages::is_explicit() && !is_hugetlbfs()) {
+ log_error(gc)("-XX:+UseLargePages (without -XX:+UseTransparentHugePages) can only be enabled "
+ "when using a %s filesystem", ZFILESYSTEM_HUGETLBFS);
+ return;
+ }
+
+ if (!ZLargePages::is_explicit() && is_hugetlbfs()) {
+ log_error(gc)("-XX:+UseLargePages must be enabled when using a %s filesystem",
+ ZFILESYSTEM_HUGETLBFS);
+ return;
+ }
+
+ const size_t expected_block_size = is_tmpfs() ? os::vm_page_size() : os::large_page_size();
+ if (expected_block_size != _block_size) {
+ log_error(gc)("%s filesystem has unexpected block size " SIZE_FORMAT " (expected " SIZE_FORMAT ")",
+ is_tmpfs() ? ZFILESYSTEM_TMPFS : ZFILESYSTEM_HUGETLBFS, _block_size, expected_block_size);
+ return;
+ }
+
+ // Successfully initialized
+ _initialized = true;
+}
+
+int ZBackingFile::create_mem_fd(const char* name) const {
+ // Create file name
+ char filename[PATH_MAX];
+ snprintf(filename, sizeof(filename), "%s%s", name, ZLargePages::is_explicit() ? ".hugetlb" : "");
+
+ // Create file
+ const int extra_flags = ZLargePages::is_explicit() ? MFD_HUGETLB : 0;
+ const int fd = z_memfd_create(filename, MFD_CLOEXEC | extra_flags);
+ if (fd == -1) {
+ ZErrno err;
+ log_debug(gc, init)("Failed to create memfd file (%s)",
+ ((ZLargePages::is_explicit() && err == EINVAL) ? "Hugepages not supported" : err.to_string()));
+ return -1;
+ }
+
+ log_info(gc, init)("Heap backed by file: /memfd:%s", filename);
+
+ return fd;
+}
+
+int ZBackingFile::create_file_fd(const char* name) const {
+ const char* const filesystem = ZLargePages::is_explicit()
+ ? ZFILESYSTEM_HUGETLBFS
+ : ZFILESYSTEM_TMPFS;
+ const char** const preferred_mountpoints = ZLargePages::is_explicit()
+ ? z_preferred_hugetlbfs_mountpoints
+ : z_preferred_tmpfs_mountpoints;
+
+ // Find mountpoint
+ ZBackingPath path(filesystem, preferred_mountpoints);
+ if (path.get() == NULL) {
+ log_error(gc)("Use -XX:ZPath to specify the path to a %s filesystem", filesystem);
+ return -1;
+ }
+
+ // Try to create an anonymous file using the O_TMPFILE flag. Note that this
+ // flag requires kernel >= 3.11. If this fails we fall back to open/unlink.
+ const int fd_anon = os::open(path.get(), O_TMPFILE|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR);
+ if (fd_anon == -1) {
+ ZErrno err;
+ log_debug(gc, init)("Failed to create anonymous file in %s (%s)", path.get(),
+ (err == EINVAL ? "Not supported" : err.to_string()));
+ } else {
+ // Get inode number for anonymous file
+ struct stat stat_buf;
+ if (fstat(fd_anon, &stat_buf) == -1) {
+ ZErrno err;
+ log_error(gc)("Failed to determine inode number for anonymous file (%s)", err.to_string());
+ return -1;
+ }
+
+ log_info(gc, init)("Heap backed by file: %s/#" UINT64_FORMAT, path.get(), (uint64_t)stat_buf.st_ino);
+
+ return fd_anon;
+ }
+
+ log_debug(gc, init)("Falling back to open/unlink");
+
+ // Create file name
+ char filename[PATH_MAX];
+ snprintf(filename, sizeof(filename), "%s/%s.%d", path.get(), name, os::current_process_id());
+
+ // Create file
+ const int fd = os::open(filename, O_CREAT|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR);
+ if (fd == -1) {
+ ZErrno err;
+ log_error(gc)("Failed to create file %s (%s)", filename, err.to_string());
+ return -1;
+ }
+
+ // Unlink file
+ if (unlink(filename) == -1) {
+ ZErrno err;
+ log_error(gc)("Failed to unlink file %s (%s)", filename, err.to_string());
+ return -1;
+ }
+
+ log_info(gc, init)("Heap backed by file: %s", filename);
+
+ return fd;
+}
+
+int ZBackingFile::create_fd(const char* name) const {
+ if (ZPath == NULL) {
+ // If the path is not explicitly specified, then we first try to create a memfd file
+ // instead of looking for a tmpfd/hugetlbfs mount point. Note that memfd_create() might
+ // not be supported at all (requires kernel >= 3.17), or it might not support large
+ // pages (requires kernel >= 4.14). If memfd_create() fails, then we try to create a
+ // file on an accessible tmpfs or hugetlbfs mount point.
+ const int fd = create_mem_fd(name);
+ if (fd != -1) {
+ return fd;
+ }
+
+ log_debug(gc, init)("Falling back to searching for an accessible mount point");
+ }
+
+ return create_file_fd(name);
+}
+
+bool ZBackingFile::is_initialized() const {
+ return _initialized;
+}
+
+int ZBackingFile::fd() const {
+ return _fd;
+}
+
+size_t ZBackingFile::size() const {
+ return _size;
+}
+
+size_t ZBackingFile::available() const {
+ return _available;
+}
+
+bool ZBackingFile::is_tmpfs() const {
+ return _filesystem == TMPFS_MAGIC;
+}
+
+bool ZBackingFile::is_hugetlbfs() const {
+ return _filesystem == HUGETLBFS_MAGIC;
+}
+
+bool ZBackingFile::tmpfs_supports_transparent_huge_pages() const {
+ // If the shmem_enabled file exists and is readable then we
+ // know the kernel supports transparent huge pages for tmpfs.
+ return access(ZFILENAME_SHMEM_ENABLED, R_OK) == 0;
+}
+
+ZErrno ZBackingFile::fallocate_compat_ftruncate(size_t size) const {
+ while (ftruncate(_fd, size) == -1) {
+ if (errno != EINTR) {
+ // Failed
+ return errno;
+ }
+ }
+
+ // Success
+ return 0;
+}
+
+ZErrno ZBackingFile::fallocate_compat_mmap(size_t offset, size_t length, bool touch) const {
+ // On hugetlbfs, mapping a file segment will fail immediately, without
+ // the need to touch the mapped pages first, if there aren't enough huge
+ // pages available to back the mapping.
+ void* const addr = mmap(0, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, offset);
+ if (addr == MAP_FAILED) {
+ // Failed
+ return errno;
+ }
+
+ // Once mapped, the huge pages are only reserved. We need to touch them
+ // to associate them with the file segment. Note that we can not punch
+ // hole in file segments which only have reserved pages.
+ if (touch) {
+ char* const start = (char*)addr;
+ char* const end = start + length;
+ os::pretouch_memory(start, end, _block_size);
+ }
+
+ // Unmap again. From now on, the huge pages that were mapped are allocated
+ // to this file. There's no risk in getting SIGBUS when touching them.
+ if (munmap(addr, length) == -1) {
+ // Failed
+ return errno;
+ }
+
+ // Success
+ return 0;
+}
+
+ZErrno ZBackingFile::fallocate_compat_pwrite(size_t offset, size_t length) const {
+ uint8_t data = 0;
+
+ // Allocate backing memory by writing to each block
+ for (size_t pos = offset; pos < offset + length; pos += _block_size) {
+ if (pwrite(_fd, &data, sizeof(data), pos) == -1) {
+ // Failed
+ return errno;
+ }
+ }
+
+ // Success
+ return 0;
+}
+
+ZErrno ZBackingFile::fallocate_fill_hole_compat(size_t offset, size_t length) {
+ // fallocate(2) is only supported by tmpfs since Linux 3.5, and by hugetlbfs
+ // since Linux 4.3. When fallocate(2) is not supported we emulate it using
+ // ftruncate/pwrite (for tmpfs) or ftruncate/mmap/munmap (for hugetlbfs).
+
+ const size_t end = offset + length;
+ if (end > _size) {
+ // Increase file size
+ const ZErrno err = fallocate_compat_ftruncate(end);
+ if (err) {
+ // Failed
+ return err;
+ }
+ }
+
+ // Allocate backing memory
+ const ZErrno err = is_hugetlbfs() ? fallocate_compat_mmap(offset, length, false /* touch */)
+ : fallocate_compat_pwrite(offset, length);
+ if (err) {
+ if (end > _size) {
+ // Restore file size
+ fallocate_compat_ftruncate(_size);
+ }
+
+ // Failed
+ return err;
+ }
+
+ if (end > _size) {
+ // Record new file size
+ _size = end;
+ }
+
+ // Success
+ return 0;
+}
+
+ZErrno ZBackingFile::fallocate_fill_hole_syscall(size_t offset, size_t length) {
+ const int mode = 0; // Allocate
+ const int res = z_fallocate(_fd, mode, offset, length);
+ if (res == -1) {
+ // Failed
+ return errno;
+ }
+
+ const size_t end = offset + length;
+ if (end > _size) {
+ // Record new file size
+ _size = end;
+ }
+
+ // Success
+ return 0;
+}
+
+ZErrno ZBackingFile::fallocate_fill_hole(size_t offset, size_t length) {
+ // Using compat mode is more efficient when allocating space on hugetlbfs.
+ // Note that allocating huge pages this way will only reserve them, and not
+ // associate them with segments of the file. We must guarantee that we at
+ // some point touch these segments, otherwise we can not punch hole in them.
+ if (z_fallocate_supported && !is_hugetlbfs()) {
+ const ZErrno err = fallocate_fill_hole_syscall(offset, length);
+ if (!err) {
+ // Success
+ return 0;
+ }
+
+ if (err != ENOSYS && err != EOPNOTSUPP) {
+ // Failed
+ return err;
+ }
+
+ // Not supported
+ log_debug(gc)("Falling back to fallocate() compatibility mode");
+ z_fallocate_supported = false;
+ }
+
+ return fallocate_fill_hole_compat(offset, length);
+}
+
+ZErrno ZBackingFile::fallocate_punch_hole(size_t offset, size_t length) {
+ if (is_hugetlbfs()) {
+ // We can only punch hole in pages that have been touched. Non-touched
+ // pages are only reserved, and not associated with any specific file
+ // segment. We don't know which pages have been previously touched, so
+ // we always touch them here to guarantee that we can punch hole.
+ const ZErrno err = fallocate_compat_mmap(offset, length, true /* touch */);
+ if (err) {
+ // Failed
+ return err;
+ }
+ }
+
+ const int mode = FALLOC_FL_PUNCH_HOLE|FALLOC_FL_KEEP_SIZE;
+ if (z_fallocate(_fd, mode, offset, length) == -1) {
+ // Failed
+ return errno;
+ }
+
+ // Success
+ return 0;
+}
+
+ZErrno ZBackingFile::split_and_fallocate(bool punch_hole, size_t offset, size_t length) {
+ // Try first half
+ const size_t offset0 = offset;
+ const size_t length0 = align_up(length / 2, _block_size);
+ const ZErrno err0 = fallocate(punch_hole, offset0, length0);
+ if (err0) {
+ return err0;
+ }
+
+ // Try second half
+ const size_t offset1 = offset0 + length0;
+ const size_t length1 = length - length0;
+ const ZErrno err1 = fallocate(punch_hole, offset1, length1);
+ if (err1) {
+ return err1;
+ }
+
+ // Success
+ return 0;
+}
+
+ZErrno ZBackingFile::fallocate(bool punch_hole, size_t offset, size_t length) {
+ assert(is_aligned(offset, _block_size), "Invalid offset");
+ assert(is_aligned(length, _block_size), "Invalid length");
+
+ const ZErrno err = punch_hole ? fallocate_punch_hole(offset, length) : fallocate_fill_hole(offset, length);
+ if (err == EINTR && length > _block_size) {
+ // Calling fallocate(2) with a large length can take a long time to
+ // complete. When running profilers, such as VTune, this syscall will
+ // be constantly interrupted by signals. Expanding the file in smaller
+ // steps avoids this problem.
+ return split_and_fallocate(punch_hole, offset, length);
+ }
+
+ return err;
+}
+
+bool ZBackingFile::commit_inner(size_t offset, size_t length) {
+ log_trace(gc, heap)("Committing memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
+ offset / M, (offset + length) / M, length / M);
+
+retry:
+ const ZErrno err = fallocate(false /* punch_hole */, offset, length);
+ if (err) {
+ if (err == ENOSPC && !is_init_completed() && is_hugetlbfs() && z_fallocate_hugetlbfs_attempts-- > 0) {
+ // If we fail to allocate during initialization, due to lack of space on
+ // the hugetlbfs filesystem, then we wait and retry a few times before
+ // giving up. Otherwise there is a risk that running JVMs back-to-back
+ // will fail, since there is a delay between process termination and the
+ // huge pages owned by that process being returned to the huge page pool
+ // and made available for new allocations.
+ log_debug(gc, init)("Failed to commit memory (%s), retrying", err.to_string());
+
+ // Wait and retry in one second, in the hope that huge pages will be
+ // available by then.
+ sleep(1);
+ goto retry;
+ }
+
+ // Failed
+ log_error(gc)("Failed to commit memory (%s)", err.to_string());
+ return false;
+ }
+
+ // Success
+ return true;
+}
+
+size_t ZBackingFile::commit(size_t offset, size_t length) {
+ // Try to commit the whole region
+ if (commit_inner(offset, length)) {
+ // Success
+ return length;
+ }
+
+ // Failed, try to commit as much as possible
+ size_t start = offset;
+ size_t end = offset + length;
+
+ for (;;) {
+ length = align_down((end - start) / 2, ZGranuleSize);
+ if (length < ZGranuleSize) {
+ // Done, don't commit more
+ return start - offset;
+ }
+
+ if (commit_inner(start, length)) {
+ // Success, try commit more
+ start += length;
+ } else {
+ // Failed, try commit less
+ end -= length;
+ }
+ }
+}
+
+size_t ZBackingFile::uncommit(size_t offset, size_t length) {
+ log_trace(gc, heap)("Uncommitting memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
+ offset / M, (offset + length) / M, length / M);
+
+ const ZErrno err = fallocate(true /* punch_hole */, offset, length);
+ if (err) {
+ log_error(gc)("Failed to uncommit memory (%s)", err.to_string());
+ return 0;
+ }
+
+ return length;
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.hpp Thu Jun 13 15:24:34 2019 +0100
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef OS_CPU_LINUX_AARCH64_GC_Z_ZBACKINGFILE_LINUX_AARCH64_HPP
+#define OS_CPU_LINUX_AARCH64_GC_Z_ZBACKINGFILE_LINUX_AARCH64_HPP
+
+#include "memory/allocation.hpp"
+
+class ZErrno;
+
+class ZBackingFile {
+private:
+ int _fd;
+ size_t _size;
+ uint64_t _filesystem;
+ size_t _block_size;
+ size_t _available;
+ bool _initialized;
+
+ int create_mem_fd(const char* name) const;
+ int create_file_fd(const char* name) const;
+ int create_fd(const char* name) const;
+
+ bool is_tmpfs() const;
+ bool is_hugetlbfs() const;
+ bool tmpfs_supports_transparent_huge_pages() const;
+
+ ZErrno fallocate_compat_ftruncate(size_t size) const;
+ ZErrno fallocate_compat_mmap(size_t offset, size_t length, bool reserve_only) const;
+ ZErrno fallocate_compat_pwrite(size_t offset, size_t length) const;
+ ZErrno fallocate_fill_hole_compat(size_t offset, size_t length);
+ ZErrno fallocate_fill_hole_syscall(size_t offset, size_t length);
+ ZErrno fallocate_fill_hole(size_t offset, size_t length);
+ ZErrno fallocate_punch_hole(size_t offset, size_t length);
+ ZErrno split_and_fallocate(bool punch_hole, size_t offset, size_t length);
+ ZErrno fallocate(bool punch_hole, size_t offset, size_t length);
+
+ bool commit_inner(size_t offset, size_t length);
+
+public:
+ ZBackingFile();
+
+ bool is_initialized() const;
+
+ int fd() const;
+ size_t size() const;
+ size_t available() const;
+
+ size_t commit(size_t offset, size_t length);
+ size_t uncommit(size_t offset, size_t length);
+};
+
+#endif // OS_CPU_LINUX_AARCH64_GC_Z_ZBACKINGFILE_LINUX_AARCH64_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingPath_linux_aarch64.cpp Thu Jun 13 15:24:34 2019 +0100
@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zArray.inline.hpp"
+#include "zBackingPath_linux_aarch64.hpp"
+#include "gc/z/zErrno.hpp"
+#include "logging/log.hpp"
+
+#include <stdio.h>
+#include <unistd.h>
+
+// Mount information, see proc(5) for more details.
+#define PROC_SELF_MOUNTINFO "/proc/self/mountinfo"
+
+ZBackingPath::ZBackingPath(const char* filesystem, const char** preferred_mountpoints) {
+ if (ZPath != NULL) {
+ // Use specified path
+ _path = strdup(ZPath);
+ } else {
+ // Find suitable path
+ _path = find_mountpoint(filesystem, preferred_mountpoints);
+ }
+}
+
+ZBackingPath::~ZBackingPath() {
+ free(_path);
+ _path = NULL;
+}
+
+char* ZBackingPath::get_mountpoint(const char* line, const char* filesystem) const {
+ char* line_mountpoint = NULL;
+ char* line_filesystem = NULL;
+
+ // Parse line and return a newly allocated string containing the mount point if
+ // the line contains a matching filesystem and the mount point is accessible by
+ // the current user.
+ if (sscanf(line, "%*u %*u %*u:%*u %*s %ms %*[^-]- %ms", &line_mountpoint, &line_filesystem) != 2 ||
+ strcmp(line_filesystem, filesystem) != 0 ||
+ access(line_mountpoint, R_OK|W_OK|X_OK) != 0) {
+ // Not a matching or accessible filesystem
+ free(line_mountpoint);
+ line_mountpoint = NULL;
+ }
+
+ free(line_filesystem);
+
+ return line_mountpoint;
+}
+
+void ZBackingPath::get_mountpoints(const char* filesystem, ZArray<char*>* mountpoints) const {
+ FILE* fd = fopen(PROC_SELF_MOUNTINFO, "r");
+ if (fd == NULL) {
+ ZErrno err;
+ log_error(gc)("Failed to open %s: %s", PROC_SELF_MOUNTINFO, err.to_string());
+ return;
+ }
+
+ char* line = NULL;
+ size_t length = 0;
+
+ while (getline(&line, &length, fd) != -1) {
+ char* const mountpoint = get_mountpoint(line, filesystem);
+ if (mountpoint != NULL) {
+ mountpoints->add(mountpoint);
+ }
+ }
+
+ free(line);
+ fclose(fd);
+}
+
+void ZBackingPath::free_mountpoints(ZArray<char*>* mountpoints) const {
+ ZArrayIterator<char*> iter(mountpoints);
+ for (char* mountpoint; iter.next(&mountpoint);) {
+ free(mountpoint);
+ }
+ mountpoints->clear();
+}
+
+char* ZBackingPath::find_preferred_mountpoint(const char* filesystem,
+ ZArray<char*>* mountpoints,
+ const char** preferred_mountpoints) const {
+ // Find preferred mount point
+ ZArrayIterator<char*> iter1(mountpoints);
+ for (char* mountpoint; iter1.next(&mountpoint);) {
+ for (const char** preferred = preferred_mountpoints; *preferred != NULL; preferred++) {
+ if (!strcmp(mountpoint, *preferred)) {
+ // Preferred mount point found
+ return strdup(mountpoint);
+ }
+ }
+ }
+
+ // Preferred mount point not found
+ log_error(gc)("More than one %s filesystem found:", filesystem);
+ ZArrayIterator<char*> iter2(mountpoints);
+ for (char* mountpoint; iter2.next(&mountpoint);) {
+ log_error(gc)(" %s", mountpoint);
+ }
+
+ return NULL;
+}
+
+char* ZBackingPath::find_mountpoint(const char* filesystem, const char** preferred_mountpoints) const {
+ char* path = NULL;
+ ZArray<char*> mountpoints;
+
+ get_mountpoints(filesystem, &mountpoints);
+
+ if (mountpoints.size() == 0) {
+ // No mount point found
+ log_error(gc)("Failed to find an accessible %s filesystem", filesystem);
+ } else if (mountpoints.size() == 1) {
+ // One mount point found
+ path = strdup(mountpoints.at(0));
+ } else {
+ // More than one mount point found
+ path = find_preferred_mountpoint(filesystem, &mountpoints, preferred_mountpoints);
+ }
+
+ free_mountpoints(&mountpoints);
+
+ return path;
+}
+
+const char* ZBackingPath::get() const {
+ return _path;
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingPath_linux_aarch64.hpp Thu Jun 13 15:24:34 2019 +0100
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef OS_CPU_LINUX_AARCH64_GC_Z_ZBACKINGPATH_LINUX_AARCH64_HPP
+#define OS_CPU_LINUX_AARCH64_GC_Z_ZBACKINGPATH_LINUX_AARCH64_HPP
+
+#include "gc/z/zArray.hpp"
+#include "memory/allocation.hpp"
+
+class ZBackingPath : public StackObj {
+private:
+ char* _path;
+
+ char* get_mountpoint(const char* line,
+ const char* filesystem) const;
+ void get_mountpoints(const char* filesystem,
+ ZArray<char*>* mountpoints) const;
+ void free_mountpoints(ZArray<char*>* mountpoints) const;
+ char* find_preferred_mountpoint(const char* filesystem,
+ ZArray<char*>* mountpoints,
+ const char** preferred_mountpoints) const;
+ char* find_mountpoint(const char* filesystem,
+ const char** preferred_mountpoints) const;
+
+public:
+ ZBackingPath(const char* filesystem, const char** preferred_mountpoints);
+ ~ZBackingPath();
+
+ const char* get() const;
+};
+
+#endif // OS_CPU_LINUX_AARCH64_GC_Z_ZBACKINGPATH_LINUX_AARCH64_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zGlobals_linux_aarch64.cpp Thu Jun 13 15:24:34 2019 +0100
@@ -0,0 +1,173 @@
+/*
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zGlobals.hpp"
+#include "gc/z/zUtils.inline.hpp"
+#include "runtime/globals.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+//
+// The heap can have three different layouts, depending on the max heap size.
+//
+// Address Space & Pointer Layout 1
+// --------------------------------
+//
+// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB)
+// . .
+// . .
+// . .
+// +--------------------------------+ 0x0000014000000000 (20TB)
+// | Remapped View |
+// +--------------------------------+ 0x0000010000000000 (16TB)
+// | (Reserved, but unused) |
+// +--------------------------------+ 0x00000c0000000000 (12TB)
+// | Marked1 View |
+// +--------------------------------+ 0x0000080000000000 (8TB)
+// | Marked0 View |
+// +--------------------------------+ 0x0000040000000000 (4TB)
+// . .
+// +--------------------------------+ 0x0000000000000000
+//
+// 6 4 4 4 4
+// 3 6 5 2 1 0
+// +--------------------+----+-----------------------------------------------+
+// |00000000 00000000 00|1111|11 11111111 11111111 11111111 11111111 11111111|
+// +--------------------+----+-----------------------------------------------+
+// | | |
+// | | * 41-0 Object Offset (42-bits, 4TB address space)
+// | |
+// | * 45-42 Metadata Bits (4-bits) 0001 = Marked0 (Address view 4-8TB)
+// | 0010 = Marked1 (Address view 8-12TB)
+// | 0100 = Remapped (Address view 16-20TB)
+// | 1000 = Finalizable (Address view N/A)
+// |
+// * 63-46 Fixed (18-bits, always zero)
+//
+//
+// Address Space & Pointer Layout 2
+// --------------------------------
+//
+// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB)
+// . .
+// . .
+// . .
+// +--------------------------------+ 0x0000280000000000 (40TB)
+// | Remapped View |
+// +--------------------------------+ 0x0000200000000000 (32TB)
+// | (Reserved, but unused) |
+// +--------------------------------+ 0x0000180000000000 (24TB)
+// | Marked1 View |
+// +--------------------------------+ 0x0000100000000000 (16TB)
+// | Marked0 View |
+// +--------------------------------+ 0x0000080000000000 (8TB)
+// . .
+// +--------------------------------+ 0x0000000000000000
+//
+// 6 4 4 4 4
+// 3 7 6 3 2 0
+// +------------------+-----+------------------------------------------------+
+// |00000000 00000000 0|1111|111 11111111 11111111 11111111 11111111 11111111|
+// +-------------------+----+------------------------------------------------+
+// | | |
+// | | * 42-0 Object Offset (43-bits, 8TB address space)
+// | |
+// | * 46-43 Metadata Bits (4-bits) 0001 = Marked0 (Address view 8-16TB)
+// | 0010 = Marked1 (Address view 16-24TB)
+// | 0100 = Remapped (Address view 32-40TB)
+// | 1000 = Finalizable (Address view N/A)
+// |
+// * 63-47 Fixed (17-bits, always zero)
+//
+//
+// Address Space & Pointer Layout 3
+// --------------------------------
+//
+// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB)
+// . .
+// . .
+// . .
+// +--------------------------------+ 0x0000500000000000 (80TB)
+// | Remapped View |
+// +--------------------------------+ 0x0000400000000000 (64TB)
+// | (Reserved, but unused) |
+// +--------------------------------+ 0x0000300000000000 (48TB)
+// | Marked1 View |
+// +--------------------------------+ 0x0000200000000000 (32TB)
+// | Marked0 View |
+// +--------------------------------+ 0x0000100000000000 (16TB)
+// . .
+// +--------------------------------+ 0x0000000000000000
+//
+// 6 4 4 4 4
+// 3 8 7 4 3 0
+// +------------------+----+-------------------------------------------------+
+// |00000000 00000000 |1111|1111 11111111 11111111 11111111 11111111 11111111|
+// +------------------+----+-------------------------------------------------+
+// | | |
+// | | * 43-0 Object Offset (44-bits, 16TB address space)
+// | |
+// | * 47-44 Metadata Bits (4-bits) 0001 = Marked0 (Address view 16-32TB)
+// | 0010 = Marked1 (Address view 32-48TB)
+// | 0100 = Remapped (Address view 64-80TB)
+// | 1000 = Finalizable (Address view N/A)
+// |
+// * 63-48 Fixed (16-bits, always zero)
+//
+
+uintptr_t ZPlatformAddressSpaceStart() {
+ const uintptr_t first_heap_view_address = (uintptr_t)1 << (ZPlatformAddressMetadataShift() + 0);
+ const size_t min_address_offset = 0;
+ return first_heap_view_address + min_address_offset;
+}
+
+uintptr_t ZPlatformAddressSpaceEnd() {
+ const uintptr_t last_heap_view_address = (uintptr_t)1 << (ZPlatformAddressMetadataShift() + 2);
+ const size_t max_address_offset = (size_t)1 << ZPlatformAddressOffsetBits();
+ return last_heap_view_address + max_address_offset;
+}
+
+uintptr_t ZPlatformAddressReservedStart() {
+ return ZPlatformAddressSpaceStart();
+}
+
+uintptr_t ZPlatformAddressReservedEnd() {
+ return ZPlatformAddressSpaceEnd();
+}
+
+uintptr_t ZPlatformAddressBase() {
+ return 0;
+}
+
+size_t ZPlatformAddressOffsetBits() {
+ const size_t min_address_offset_bits = 42; // 4TB
+ const size_t max_address_offset_bits = 44; // 16TB
+ const size_t virtual_to_physical_ratio = 7; // 7:1
+ const size_t address_offset = ZUtils::round_up_power_of_2(MaxHeapSize * virtual_to_physical_ratio);
+ const size_t address_offset_bits = log2_intptr(address_offset);
+ return MIN2(MAX2(address_offset_bits, min_address_offset_bits), max_address_offset_bits);
+}
+
+size_t ZPlatformAddressMetadataShift() {
+ return ZPlatformAddressOffsetBits();
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zGlobals_linux_aarch64.hpp Thu Jun 13 15:24:34 2019 +0100
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef OS_CPU_LINUX_AARCH64_GC_Z_ZGLOBALS_LINUX_AARCH64_HPP
+#define OS_CPU_LINUX_AARCH64_GC_Z_ZGLOBALS_LINUX_AARCH64_HPP
+
+//
+// Page Allocation Tiers
+// ---------------------
+//
+// Page Type Page Size Object Size Limit Object Alignment
+// ------------------------------------------------------------------
+// Small 2M <= 265K <MinObjAlignmentInBytes>
+// Medium 32M <= 4M 4K
+// Large X*M > 4M 2M
+// ------------------------------------------------------------------
+//
+const size_t ZPlatformGranuleSizeShift = 21; // 2MB
+const size_t ZPlatformMaxHeapSizeShift = 46; // 16TB
+const size_t ZPlatformNMethodDisarmedOffset = 4;
+const size_t ZPlatformCacheLineSize = 64;
+
+uintptr_t ZPlatformAddressSpaceStart();
+uintptr_t ZPlatformAddressSpaceEnd();
+uintptr_t ZPlatformAddressReservedStart();
+uintptr_t ZPlatformAddressReservedEnd();
+uintptr_t ZPlatformAddressBase();
+size_t ZPlatformAddressOffsetBits();
+size_t ZPlatformAddressMetadataShift();
+
+#endif // OS_CPU_LINUX_AARCH64_GC_Z_ZGLOBALS_LINUX_AARCH64_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zPhysicalMemoryBacking_linux_aarch64.cpp Thu Jun 13 15:24:34 2019 +0100
@@ -0,0 +1,333 @@
+/*
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zBackingFile_linux_aarch64.hpp"
+#include "gc/z/zErrno.hpp"
+#include "gc/z/zGlobals.hpp"
+#include "gc/z/zLargePages.inline.hpp"
+#include "gc/z/zMemory.hpp"
+#include "gc/z/zNUMA.hpp"
+#include "gc/z/zPhysicalMemory.inline.hpp"
+#include "gc/z/zPhysicalMemoryBacking_linux_aarch64.hpp"
+#include "logging/log.hpp"
+#include "runtime/init.hpp"
+#include "runtime/os.hpp"
+#include "utilities/align.hpp"
+#include "utilities/debug.hpp"
+
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+
+//
+// Support for building on older Linux systems
+//
+
+// madvise(2) flags
+#ifndef MADV_HUGEPAGE
+#define MADV_HUGEPAGE 14
+#endif
+
+// Proc file entry for max map mount
+#define ZFILENAME_PROC_MAX_MAP_COUNT "/proc/sys/vm/max_map_count"
+
+bool ZPhysicalMemoryBacking::is_initialized() const {
+ return _file.is_initialized();
+}
+
+void ZPhysicalMemoryBacking::warn_available_space(size_t max) const {
+ // Note that the available space on a tmpfs or a hugetlbfs filesystem
+ // will be zero if no size limit was specified when it was mounted.
+ const size_t available = _file.available();
+ if (available == 0) {
+ // No size limit set, skip check
+ log_info(gc, init)("Available space on backing filesystem: N/A");
+ return;
+ }
+
+ log_info(gc, init)("Available space on backing filesystem: " SIZE_FORMAT "M", available / M);
+
+ // Warn if the filesystem doesn't currently have enough space available to hold
+ // the max heap size. The max heap size will be capped if we later hit this limit
+ // when trying to expand the heap.
+ if (available < max) {
+ log_warning(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****");
+ log_warning(gc)("Not enough space available on the backing filesystem to hold the current max Java heap");
+ log_warning(gc)("size (" SIZE_FORMAT "M). Please adjust the size of the backing filesystem accordingly "
+ "(available", max / M);
+ log_warning(gc)("space is currently " SIZE_FORMAT "M). Continuing execution with the current filesystem "
+ "size could", available / M);
+ log_warning(gc)("lead to a premature OutOfMemoryError being thrown, due to failure to map memory.");
+ }
+}
+
+void ZPhysicalMemoryBacking::warn_max_map_count(size_t max) const {
+ const char* const filename = ZFILENAME_PROC_MAX_MAP_COUNT;
+ FILE* const file = fopen(filename, "r");
+ if (file == NULL) {
+ // Failed to open file, skip check
+ log_debug(gc, init)("Failed to open %s", filename);
+ return;
+ }
+
+ size_t actual_max_map_count = 0;
+ const int result = fscanf(file, SIZE_FORMAT, &actual_max_map_count);
+ fclose(file);
+ if (result != 1) {
+ // Failed to read file, skip check
+ log_debug(gc, init)("Failed to read %s", filename);
+ return;
+ }
+
+ // The required max map count is impossible to calculate exactly since subsystems
+ // other than ZGC are also creating memory mappings, and we have no control over that.
+ // However, ZGC tends to create the most mappings and dominate the total count.
+ // In the worst cases, ZGC will map each granule three times, i.e. once per heap view.
+ // We speculate that we need another 20% to allow for non-ZGC subsystems to map memory.
+ const size_t required_max_map_count = (max / ZGranuleSize) * 3 * 1.2;
+ if (actual_max_map_count < required_max_map_count) {
+ log_warning(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****");
+ log_warning(gc)("The system limit on number of memory mappings per process might be too low for the given");
+ log_warning(gc)("max Java heap size (" SIZE_FORMAT "M). Please adjust %s to allow for at",
+ max / M, filename);
+ log_warning(gc)("least " SIZE_FORMAT " mappings (current limit is " SIZE_FORMAT "). Continuing execution "
+ "with the current", required_max_map_count, actual_max_map_count);
+ log_warning(gc)("limit could lead to a fatal error, due to failure to map memory.");
+ }
+}
+
+void ZPhysicalMemoryBacking::warn_commit_limits(size_t max) const {
+ // Warn if available space is too low
+ warn_available_space(max);
+
+ // Warn if max map count is too low
+ warn_max_map_count(max);
+}
+
+bool ZPhysicalMemoryBacking::supports_uncommit() {
+ assert(!is_init_completed(), "Invalid state");
+ assert(_file.size() >= ZGranuleSize, "Invalid size");
+
+ // Test if uncommit is supported by uncommitting and then re-committing a granule
+ return commit(uncommit(ZGranuleSize)) == ZGranuleSize;
+}
+
+size_t ZPhysicalMemoryBacking::commit(size_t size) {
+ size_t committed = 0;
+
+ // Fill holes in the backing file
+ while (committed < size) {
+ size_t allocated = 0;
+ const size_t remaining = size - committed;
+ const uintptr_t start = _uncommitted.alloc_from_front_at_most(remaining, &allocated);
+ if (start == UINTPTR_MAX) {
+ // No holes to commit
+ break;
+ }
+
+ // Try commit hole
+ const size_t filled = _file.commit(start, allocated);
+ if (filled > 0) {
+ // Successful or partialy successful
+ _committed.free(start, filled);
+ committed += filled;
+ }
+ if (filled < allocated) {
+ // Failed or partialy failed
+ _uncommitted.free(start + filled, allocated - filled);
+ return committed;
+ }
+ }
+
+ // Expand backing file
+ if (committed < size) {
+ const size_t remaining = size - committed;
+ const uintptr_t start = _file.size();
+ const size_t expanded = _file.commit(start, remaining);
+ if (expanded > 0) {
+ // Successful or partialy successful
+ _committed.free(start, expanded);
+ committed += expanded;
+ }
+ }
+
+ return committed;
+}
+
+size_t ZPhysicalMemoryBacking::uncommit(size_t size) {
+ size_t uncommitted = 0;
+
+ // Punch holes in backing file
+ while (uncommitted < size) {
+ size_t allocated = 0;
+ const size_t remaining = size - uncommitted;
+ const uintptr_t start = _committed.alloc_from_back_at_most(remaining, &allocated);
+ assert(start != UINTPTR_MAX, "Allocation should never fail");
+
+ // Try punch hole
+ const size_t punched = _file.uncommit(start, allocated);
+ if (punched > 0) {
+ // Successful or partialy successful
+ _uncommitted.free(start, punched);
+ uncommitted += punched;
+ }
+ if (punched < allocated) {
+ // Failed or partialy failed
+ _committed.free(start + punched, allocated - punched);
+ return uncommitted;
+ }
+ }
+
+ return uncommitted;
+}
+
+ZPhysicalMemory ZPhysicalMemoryBacking::alloc(size_t size) {
+ assert(is_aligned(size, ZGranuleSize), "Invalid size");
+
+ ZPhysicalMemory pmem;
+
+ // Allocate segments
+ for (size_t allocated = 0; allocated < size; allocated += ZGranuleSize) {
+ const uintptr_t start = _committed.alloc_from_front(ZGranuleSize);
+ assert(start != UINTPTR_MAX, "Allocation should never fail");
+ pmem.add_segment(ZPhysicalMemorySegment(start, ZGranuleSize));
+ }
+
+ return pmem;
+}
+
+void ZPhysicalMemoryBacking::free(const ZPhysicalMemory& pmem) {
+ const size_t nsegments = pmem.nsegments();
+
+ // Free segments
+ for (size_t i = 0; i < nsegments; i++) {
+ const ZPhysicalMemorySegment& segment = pmem.segment(i);
+ _committed.free(segment.start(), segment.size());
+ }
+}
+
+void ZPhysicalMemoryBacking::map_failed(ZErrno err) const {
+ if (err == ENOMEM) {
+ fatal("Failed to map memory. Please check the system limit on number of "
+ "memory mappings allowed per process (see %s)", ZFILENAME_PROC_MAX_MAP_COUNT);
+ } else {
+ fatal("Failed to map memory (%s)", err.to_string());
+ }
+}
+
+void ZPhysicalMemoryBacking::advise_view(uintptr_t addr, size_t size, int advice) const {
+ if (madvise((void*)addr, size, advice) == -1) {
+ ZErrno err;
+ log_error(gc)("Failed to advise on memory (advice %d, %s)", advice, err.to_string());
+ }
+}
+
+void ZPhysicalMemoryBacking::pretouch_view(uintptr_t addr, size_t size) const {
+ const size_t page_size = ZLargePages::is_explicit() ? os::large_page_size() : os::vm_page_size();
+ os::pretouch_memory((void*)addr, (void*)(addr + size), page_size);
+}
+
+void ZPhysicalMemoryBacking::map_view(const ZPhysicalMemory& pmem, uintptr_t addr, bool pretouch) const {
+ const size_t nsegments = pmem.nsegments();
+ size_t size = 0;
+
+ // Map segments
+ for (size_t i = 0; i < nsegments; i++) {
+ const ZPhysicalMemorySegment& segment = pmem.segment(i);
+ const uintptr_t segment_addr = addr + size;
+ const void* const res = mmap((void*)segment_addr, segment.size(), PROT_READ|PROT_WRITE, MAP_FIXED|MAP_SHARED, _file.fd(), segment.start());
+ if (res == MAP_FAILED) {
+ ZErrno err;
+ map_failed(err);
+ }
+
+ size += segment.size();
+ }
+
+ // Advise on use of transparent huge pages before touching it
+ if (ZLargePages::is_transparent()) {
+ advise_view(addr, size, MADV_HUGEPAGE);
+ }
+
+ // NUMA interleave memory before touching it
+ ZNUMA::memory_interleave(addr, size);
+
+ // Pre-touch memory
+ if (pretouch) {
+ pretouch_view(addr, size);
+ }
+}
+
+void ZPhysicalMemoryBacking::unmap_view(const ZPhysicalMemory& pmem, uintptr_t addr) const {
+ // Note that we must keep the address space reservation intact and just detach
+ // the backing memory. For this reason we map a new anonymous, non-accessible
+ // and non-reserved page over the mapping instead of actually unmapping.
+ const void* const res = mmap((void*)addr, pmem.size(), PROT_NONE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0);
+ if (res == MAP_FAILED) {
+ ZErrno err;
+ map_failed(err);
+ }
+}
+
+uintptr_t ZPhysicalMemoryBacking::nmt_address(uintptr_t offset) const {
+ // From an NMT point of view we treat the first heap view (marked0) as committed
+ return ZAddress::marked0(offset);
+}
+
+void ZPhysicalMemoryBacking::map(const ZPhysicalMemory& pmem, uintptr_t offset) const {
+ if (ZVerifyViews) {
+ // Map good view
+ map_view(pmem, ZAddress::good(offset), AlwaysPreTouch);
+ } else {
+ // Map all views
+ map_view(pmem, ZAddress::marked0(offset), AlwaysPreTouch);
+ map_view(pmem, ZAddress::marked1(offset), AlwaysPreTouch);
+ map_view(pmem, ZAddress::remapped(offset), AlwaysPreTouch);
+ }
+}
+
+void ZPhysicalMemoryBacking::unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const {
+ if (ZVerifyViews) {
+ // Unmap good view
+ unmap_view(pmem, ZAddress::good(offset));
+ } else {
+ // Unmap all views
+ unmap_view(pmem, ZAddress::marked0(offset));
+ unmap_view(pmem, ZAddress::marked1(offset));
+ unmap_view(pmem, ZAddress::remapped(offset));
+ }
+}
+
+void ZPhysicalMemoryBacking::debug_map(const ZPhysicalMemory& pmem, uintptr_t offset) const {
+ // Map good view
+ assert(ZVerifyViews, "Should be enabled");
+ map_view(pmem, ZAddress::good(offset), false /* pretouch */);
+}
+
+void ZPhysicalMemoryBacking::debug_unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const {
+ // Unmap good view
+ assert(ZVerifyViews, "Should be enabled");
+ unmap_view(pmem, ZAddress::good(offset));
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/os_cpu/linux_aarch64/gc/z/zPhysicalMemoryBacking_linux_aarch64.hpp Thu Jun 13 15:24:34 2019 +0100
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef OS_CPU_LINUX_AARCH64_GC_Z_ZPHYSICALMEMORYBACKING_LINUX_AARCH64_HPP
+#define OS_CPU_LINUX_AARCH64_GC_Z_ZPHYSICALMEMORYBACKING_LINUX_AARCH64_HPP
+
+#include "gc/z/zBackingFile_linux_aarch64.hpp"
+#include "gc/z/zMemory.hpp"
+
+class ZErrno;
+class ZPhysicalMemory;
+
+class ZPhysicalMemoryBacking {
+private:
+ ZBackingFile _file;
+ ZMemoryManager _committed;
+ ZMemoryManager _uncommitted;
+
+ void warn_available_space(size_t max) const;
+ void warn_max_map_count(size_t max) const;
+
+ void map_failed(ZErrno err) const;
+
+ void advise_view(uintptr_t addr, size_t size, int advice) const;
+ void pretouch_view(uintptr_t addr, size_t size) const;
+ void map_view(const ZPhysicalMemory& pmem, uintptr_t addr, bool pretouch) const;
+ void unmap_view(const ZPhysicalMemory& pmem, uintptr_t addr) const;
+
+public:
+ bool is_initialized() const;
+
+ void warn_commit_limits(size_t max) const;
+ bool supports_uncommit();
+
+ size_t commit(size_t size);
+ size_t uncommit(size_t size);
+
+ ZPhysicalMemory alloc(size_t size);
+ void free(const ZPhysicalMemory& pmem);
+
+ uintptr_t nmt_address(uintptr_t offset) const;
+
+ void map(const ZPhysicalMemory& pmem, uintptr_t offset) const;
+ void unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const;
+
+ void debug_map(const ZPhysicalMemory& pmem, uintptr_t offset) const;
+ void debug_unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const;
+};
+
+#endif // OS_CPU_LINUX_AARCH64_GC_Z_ZPHYSICALMEMORYBACKING_LINUX_AARCH64_HPP