/*

 * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "jni.h"

#include "jvm.h"

#include "classfile/classFileStream.hpp"

#include "classfile/classLoader.hpp"

#include "classfile/vmSymbols.hpp"

#include "jfr/jfrEvents.hpp"

#include "memory/allocation.inline.hpp"

#include "memory/resourceArea.hpp"

#include "oops/access.inline.hpp"

#include "oops/objArrayOop.inline.hpp"

#include "oops/oop.inline.hpp"

#include "oops/typeArrayOop.inline.hpp"

#include "prims/unsafe.hpp"

#include "runtime/atomic.hpp"

#include "runtime/globals.hpp"

#include "runtime/handles.inline.hpp"

#include "runtime/interfaceSupport.inline.hpp"

#include "runtime/jniHandles.inline.hpp"

#include "runtime/orderAccess.hpp"

#include "runtime/reflection.hpp"

#include "runtime/sharedRuntime.hpp"

#include "runtime/thread.hpp"

#include "runtime/threadSMR.hpp"

#include "runtime/vm_version.hpp"

#include "services/threadService.hpp"

#include "utilities/align.hpp"

#include "utilities/copy.hpp"

#include "utilities/dtrace.hpp"

#include "utilities/macros.hpp"

/**

 * Implementation of the jdk.internal.misc.Unsafe class

 */

#define MAX_OBJECT_SIZE \

  ( arrayOopDesc::header_size(T_DOUBLE) * HeapWordSize \

    + ((julong)max_jint * sizeof(double)) )

#define UNSAFE_ENTRY(result_type, header) \

  JVM_ENTRY(static result_type, header)

#define UNSAFE_LEAF(result_type, header) \

  JVM_LEAF(static result_type, header)

#define UNSAFE_END JVM_END

static inline void* addr_from_java(jlong addr) {

  // This assert fails in a variety of ways on 32-bit systems.

  // It is impossible to predict whether native code that converts

  // pointers to longs will sign-extend or zero-extend the addresses.

  //assert(addr == (uintptr_t)addr, "must not be odd high bits");

  return (void*)(uintptr_t)addr;

}

static inline jlong addr_to_java(void* p) {

  assert(p == (void*)(uintptr_t)p, "must not be odd high bits");

  return (uintptr_t)p;

}

// Note: The VM's obj_field and related accessors use byte-scaled

// ("unscaled") offsets, just as the unsafe methods do.

// However, the method Unsafe.fieldOffset explicitly declines to

// guarantee this.  The field offset values manipulated by the Java user

// through the Unsafe API are opaque cookies that just happen to be byte

// offsets.  We represent this state of affairs by passing the cookies

// through conversion functions when going between the VM and the Unsafe API.

// The conversion functions just happen to be no-ops at present.

static inline jlong field_offset_to_byte_offset(jlong field_offset) {

  return field_offset;

}

static inline jlong field_offset_from_byte_offset(jlong byte_offset) {

  return byte_offset;

}

static inline void assert_field_offset_sane(oop p, jlong field_offset) {

#ifdef ASSERT

  jlong byte_offset = field_offset_to_byte_offset(field_offset);

  if (p != NULL) {

    assert(byte_offset >= 0 && byte_offset <= (jlong)MAX_OBJECT_SIZE, "sane offset");

    if (byte_offset == (jint)byte_offset) {

      void* ptr_plus_disp = (address)p + byte_offset;

      assert(p->field_addr_raw((jint)byte_offset) == ptr_plus_disp,

             "raw [ptr+disp] must be consistent with oop::field_addr_raw");

    }

    jlong p_size = HeapWordSize * (jlong)(p->size());

    assert(byte_offset < p_size, "Unsafe access: offset " INT64_FORMAT " > object's size " INT64_FORMAT, (int64_t)byte_offset, (int64_t)p_size);

  }

#endif

}

static inline void* index_oop_from_field_offset_long(oop p, jlong field_offset) {

  assert_field_offset_sane(p, field_offset);

  jlong byte_offset = field_offset_to_byte_offset(field_offset);

  if (p != NULL) {

    p = Access<>::resolve(p);

  }

  if (sizeof(char*) == sizeof(jint)) {   // (this constant folds!)

    return (address)p + (jint) byte_offset;

  } else {

    return (address)p +        byte_offset;

  }

}

// Externally callable versions:

// (Use these in compiler intrinsics which emulate unsafe primitives.)

jlong Unsafe_field_offset_to_byte_offset(jlong field_offset) {

  return field_offset;

}

jlong Unsafe_field_offset_from_byte_offset(jlong byte_offset) {

  return byte_offset;

}

///// Data read/writes on the Java heap and in native (off-heap) memory

/**

 * Helper class to wrap memory accesses in JavaThread::doing_unsafe_access()

 */

class GuardUnsafeAccess {

  JavaThread* _thread;

public:

  GuardUnsafeAccess(JavaThread* thread) : _thread(thread) {

    // native/off-heap access which may raise SIGBUS if accessing

    // memory mapped file data in a region of the file which has

    // been truncated and is now invalid.

    _thread->set_doing_unsafe_access(true);

  }

  ~GuardUnsafeAccess() {

    _thread->set_doing_unsafe_access(false);

  }

};

/**

 * Helper class for accessing memory.

 *

 * Normalizes values and wraps accesses in

 * JavaThread::doing_unsafe_access() if needed.

 */

template <typename T>

class MemoryAccess : StackObj {

  JavaThread* _thread;

  oop _obj;

  ptrdiff_t _offset;

  // Resolves and returns the address of the memory access.

  // This raw memory access may fault, so we make sure it happens within the

  // guarded scope by making the access volatile at least. Since the store

  // of Thread::set_doing_unsafe_access() is also volatile, these accesses

  // can not be reordered by the compiler. Therefore, if the access triggers

  // a fault, we will know that Thread::doing_unsafe_access() returns true.

  volatile T* addr() {

    void* addr = index_oop_from_field_offset_long(_obj, _offset);

    return static_cast<volatile T*>(addr);

  }

  template <typename U>

  U normalize_for_write(U x) {

    return x;

  }

  jboolean normalize_for_write(jboolean x) {

    return x & 1;

  }

  template <typename U>

  U normalize_for_read(U x) {

    return x;

  }

  jboolean normalize_for_read(jboolean x) {

    return x != 0;

  }

public:

  MemoryAccess(JavaThread* thread, jobject obj, jlong offset)

    : _thread(thread), _obj(JNIHandles::resolve(obj)), _offset((ptrdiff_t)offset) {

    assert_field_offset_sane(_obj, offset);

  }

  T get() {

    if (_obj == NULL) {

      GuardUnsafeAccess guard(_thread);

      T ret = RawAccess<>::load(addr());

      return normalize_for_read(ret);

    } else {

      T ret = HeapAccess<>::load_at(_obj, _offset);

      return normalize_for_read(ret);

    }

  }

  void put(T x) {

    if (_obj == NULL) {

      GuardUnsafeAccess guard(_thread);

      RawAccess<>::store(addr(), normalize_for_write(x));

    } else {

      HeapAccess<>::store_at(_obj, _offset, normalize_for_write(x));

    }

  }

  T get_volatile() {

    if (_obj == NULL) {

      GuardUnsafeAccess guard(_thread);

      volatile T ret = RawAccess<MO_SEQ_CST>::load(addr());

      return normalize_for_read(ret);

    } else {

      T ret = HeapAccess<MO_SEQ_CST>::load_at(_obj, _offset);

      return normalize_for_read(ret);

    }

  }

  void put_volatile(T x) {

    if (_obj == NULL) {

      GuardUnsafeAccess guard(_thread);

      RawAccess<MO_SEQ_CST>::store(addr(), normalize_for_write(x));

    } else {

      HeapAccess<MO_SEQ_CST>::store_at(_obj, _offset, normalize_for_write(x));

    }

  }

};

// These functions allow a null base pointer with an arbitrary address.

// But if the base pointer is non-null, the offset should make some sense.

// That is, it should be in the range [0, MAX_OBJECT_SIZE].

UNSAFE_ENTRY(jobject, Unsafe_GetReference(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) {

  oop p = JNIHandles::resolve(obj);

  assert_field_offset_sane(p, offset);

  oop v = HeapAccess<ON_UNKNOWN_OOP_REF>::oop_load_at(p, offset);

  return JNIHandles::make_local(env, v);

} UNSAFE_END

UNSAFE_ENTRY(void, Unsafe_PutReference(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject x_h)) {

  oop x = JNIHandles::resolve(x_h);

  oop p = JNIHandles::resolve(obj);

  assert_field_offset_sane(p, offset);

  HeapAccess<ON_UNKNOWN_OOP_REF>::oop_store_at(p, offset, x);

} UNSAFE_END

UNSAFE_ENTRY(jobject, Unsafe_GetReferenceVolatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) {

  oop p = JNIHandles::resolve(obj);

  assert_field_offset_sane(p, offset);

  oop v = HeapAccess<MO_SEQ_CST | ON_UNKNOWN_OOP_REF>::oop_load_at(p, offset);

  return JNIHandles::make_local(env, v);

} UNSAFE_END

UNSAFE_ENTRY(void, Unsafe_PutReferenceVolatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject x_h)) {

  oop x = JNIHandles::resolve(x_h);

  oop p = JNIHandles::resolve(obj);

  assert_field_offset_sane(p, offset);

  HeapAccess<MO_SEQ_CST | ON_UNKNOWN_OOP_REF>::oop_store_at(p, offset, x);

} UNSAFE_END

UNSAFE_ENTRY(jobject, Unsafe_GetUncompressedObject(JNIEnv *env, jobject unsafe, jlong addr)) {

  oop v = *(oop*) (address) addr;

  return JNIHandles::make_local(env, v);

} UNSAFE_END

#define DEFINE_GETSETOOP(java_type, Type) \

 \

UNSAFE_ENTRY(java_type, Unsafe_Get##Type(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) { \

  return MemoryAccess<java_type>(thread, obj, offset).get(); \

} UNSAFE_END \

 \

UNSAFE_ENTRY(void, Unsafe_Put##Type(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, java_type x)) { \

  MemoryAccess<java_type>(thread, obj, offset).put(x); \

} UNSAFE_END \

 \

// END DEFINE_GETSETOOP.

DEFINE_GETSETOOP(jboolean, Boolean)

DEFINE_GETSETOOP(jbyte, Byte)

DEFINE_GETSETOOP(jshort, Short);

DEFINE_GETSETOOP(jchar, Char);

DEFINE_GETSETOOP(jint, Int);

DEFINE_GETSETOOP(jlong, Long);

DEFINE_GETSETOOP(jfloat, Float);

DEFINE_GETSETOOP(jdouble, Double);

#undef DEFINE_GETSETOOP

#define DEFINE_GETSETOOP_VOLATILE(java_type, Type) \

 \

UNSAFE_ENTRY(java_type, Unsafe_Get##Type##Volatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) { \

  return MemoryAccess<java_type>(thread, obj, offset).get_volatile(); \

} UNSAFE_END \

 \

UNSAFE_ENTRY(void, Unsafe_Put##Type##Volatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, java_type x)) { \

  MemoryAccess<java_type>(thread, obj, offset).put_volatile(x); \

} UNSAFE_END \

 \

// END DEFINE_GETSETOOP_VOLATILE.

DEFINE_GETSETOOP_VOLATILE(jboolean, Boolean)

DEFINE_GETSETOOP_VOLATILE(jbyte, Byte)

DEFINE_GETSETOOP_VOLATILE(jshort, Short);

DEFINE_GETSETOOP_VOLATILE(jchar, Char);

DEFINE_GETSETOOP_VOLATILE(jint, Int);

DEFINE_GETSETOOP_VOLATILE(jlong, Long);

DEFINE_GETSETOOP_VOLATILE(jfloat, Float);

DEFINE_GETSETOOP_VOLATILE(jdouble, Double);

#undef DEFINE_GETSETOOP_VOLATILE

UNSAFE_LEAF(void, Unsafe_LoadFence(JNIEnv *env, jobject unsafe)) {

  OrderAccess::acquire();

} UNSAFE_END

UNSAFE_LEAF(void, Unsafe_StoreFence(JNIEnv *env, jobject unsafe)) {

  OrderAccess::release();

} UNSAFE_END

UNSAFE_LEAF(void, Unsafe_FullFence(JNIEnv *env, jobject unsafe)) {

  OrderAccess::fence();

} UNSAFE_END

////// Allocation requests

UNSAFE_ENTRY(jobject, Unsafe_AllocateInstance(JNIEnv *env, jobject unsafe, jclass cls)) {

  ThreadToNativeFromVM ttnfv(thread);

  return env->AllocObject(cls);

} UNSAFE_END

UNSAFE_ENTRY(jlong, Unsafe_AllocateMemory0(JNIEnv *env, jobject unsafe, jlong size)) {

  size_t sz = (size_t)size;

  sz = align_up(sz, HeapWordSize);

  void* x = os::malloc(sz, mtOther);

  return addr_to_java(x);

} UNSAFE_END

UNSAFE_ENTRY(jlong, Unsafe_ReallocateMemory0(JNIEnv *env, jobject unsafe, jlong addr, jlong size)) {

  void* p = addr_from_java(addr);

  size_t sz = (size_t)size;

  sz = align_up(sz, HeapWordSize);

  void* x = os::realloc(p, sz, mtOther);

  return addr_to_java(x);

} UNSAFE_END

UNSAFE_ENTRY(void, Unsafe_FreeMemory0(JNIEnv *env, jobject unsafe, jlong addr)) {

  void* p = addr_from_java(addr);

  os::free(p);

} UNSAFE_END

UNSAFE_ENTRY(void, Unsafe_SetMemory0(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jlong size, jbyte value)) {

  size_t sz = (size_t)size;

  oop base = JNIHandles::resolve(obj);

  void* p = index_oop_from_field_offset_long(base, offset);

  Copy::fill_to_memory_atomic(p, sz, value);

} UNSAFE_END

UNSAFE_ENTRY(void, Unsafe_CopyMemory0(JNIEnv *env, jobject unsafe, jobject srcObj, jlong srcOffset, jobject dstObj, jlong dstOffset, jlong size)) {

  size_t sz = (size_t)size;

  oop srcp = JNIHandles::resolve(srcObj);

  oop dstp = JNIHandles::resolve(dstObj);

  void* src = index_oop_from_field_offset_long(srcp, srcOffset);

  void* dst = index_oop_from_field_offset_long(dstp, dstOffset);

  {

    GuardUnsafeAccess guard(thread);

    if (StubRoutines::unsafe_arraycopy() != NULL) {

      StubRoutines::UnsafeArrayCopy_stub()(src, dst, sz);

    } else {

      Copy::conjoint_memory_atomic(src, dst, sz);

    }

  }

} UNSAFE_END

// This function is a leaf since if the source and destination are both in native memory

// the copy may potentially be very large, and we don't want to disable GC if we can avoid it.

// If either source or destination (or both) are on the heap, the function will enter VM using

// JVM_ENTRY_FROM_LEAF

UNSAFE_LEAF(void, Unsafe_CopySwapMemory0(JNIEnv *env, jobject unsafe, jobject srcObj, jlong srcOffset, jobject dstObj, jlong dstOffset, jlong size, jlong elemSize)) {

  size_t sz = (size_t)size;

  size_t esz = (size_t)elemSize;

  if (srcObj == NULL && dstObj == NULL) {

    // Both src & dst are in native memory

    address src = (address)srcOffset;

    address dst = (address)dstOffset;

    {

      JavaThread* thread = JavaThread::thread_from_jni_environment(env);

      GuardUnsafeAccess guard(thread);

      Copy::conjoint_swap(src, dst, sz, esz);

    }

  } else {

    // At least one of src/dst are on heap, transition to VM to access raw pointers

    JVM_ENTRY_FROM_LEAF(env, void, Unsafe_CopySwapMemory0) {

      oop srcp = JNIHandles::resolve(srcObj);

      oop dstp = JNIHandles::resolve(dstObj);

      address src = (address)index_oop_from_field_offset_long(srcp, srcOffset);

      address dst = (address)index_oop_from_field_offset_long(dstp, dstOffset);

      {

        GuardUnsafeAccess guard(thread);

        Copy::conjoint_swap(src, dst, sz, esz);

      }

    } JVM_END

  }

} UNSAFE_END

UNSAFE_LEAF (void, Unsafe_WriteBack0(JNIEnv *env, jobject unsafe, jlong line)) {

  assert(VM_Version::supports_data_cache_line_flush(), "should not get here");

#ifdef ASSERT

  if (TraceMemoryWriteback) {

    tty->print_cr("Unsafe: writeback 0x%p", addr_from_java(line));

  }

#endif

  assert(StubRoutines::data_cache_writeback() != NULL, "sanity");

  (StubRoutines::DataCacheWriteback_stub())(addr_from_java(line));

} UNSAFE_END

static void doWriteBackSync0(bool is_pre)

{

  assert(StubRoutines::data_cache_writeback_sync() != NULL, "sanity");

  (StubRoutines::DataCacheWritebackSync_stub())(is_pre);

}

UNSAFE_LEAF (void, Unsafe_WriteBackPreSync0(JNIEnv *env, jobject unsafe)) {

  assert(VM_Version::supports_data_cache_line_flush(), "should not get here");

#ifdef ASSERT

  if (TraceMemoryWriteback) {

      tty->print_cr("Unsafe: writeback pre-sync");

  }

#endif

  doWriteBackSync0(true);

} UNSAFE_END

UNSAFE_LEAF (void, Unsafe_WriteBackPostSync0(JNIEnv *env, jobject unsafe)) {

  assert(VM_Version::supports_data_cache_line_flush(), "should not get here");

#ifdef ASSERT

  if (TraceMemoryWriteback) {

    tty->print_cr("Unsafe: writeback pre-sync");

  }

#endif

  doWriteBackSync0(false);

} UNSAFE_END

////// Random queries