/*

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "jni.h"

#include "jvm.h"

#include "classfile/classFileStream.hpp"

#include "classfile/vmSymbols.hpp"

#include "jfr/jfrEvents.hpp"

#include "memory/allocation.inline.hpp"

#include "memory/resourceArea.hpp"

#include "oops/access.inline.hpp"

#include "oops/fieldStreams.hpp"

#include "oops/objArrayOop.inline.hpp"

#include "oops/oop.inline.hpp"

#include "oops/typeArrayOop.inline.hpp"

#include "prims/unsafe.hpp"

#include "runtime/atomic.hpp"

#include "runtime/globals.hpp"

#include "runtime/interfaceSupport.inline.hpp"

#include "runtime/jniHandles.inline.hpp"

#include "runtime/orderAccess.hpp"

#include "runtime/reflection.hpp"

#include "runtime/thread.hpp"

#include "runtime/threadSMR.hpp"

#include "runtime/vm_version.hpp"

#include "services/threadService.hpp"

#include "utilities/align.hpp"

#include "utilities/copy.hpp"

#include "utilities/dtrace.hpp"

#include "utilities/macros.hpp"

/**

 * Implementation of the jdk.internal.misc.Unsafe class

 */

#define MAX_OBJECT_SIZE \

  ( arrayOopDesc::header_size(T_DOUBLE) * HeapWordSize \

    + ((julong)max_jint * sizeof(double)) )

#define UNSAFE_ENTRY(result_type, header) \

  JVM_ENTRY(static result_type, header)

#define UNSAFE_LEAF(result_type, header) \

  JVM_LEAF(static result_type, header)

#define UNSAFE_END JVM_END

static inline void* addr_from_java(jlong addr) {

  // This assert fails in a variety of ways on 32-bit systems.

  // It is impossible to predict whether native code that converts

  // pointers to longs will sign-extend or zero-extend the addresses.

  //assert(addr == (uintptr_t)addr, "must not be odd high bits");

  return (void*)(uintptr_t)addr;

}

static inline jlong addr_to_java(void* p) {

  assert(p == (void*)(uintptr_t)p, "must not be odd high bits");

  return (uintptr_t)p;

}

// Note: The VM's obj_field and related accessors use byte-scaled

// ("unscaled") offsets, just as the unsafe methods do.

// However, the method Unsafe.fieldOffset explicitly declines to

// guarantee this.  The field offset values manipulated by the Java user

// through the Unsafe API are opaque cookies that just happen to be byte

// offsets.  We represent this state of affairs by passing the cookies

// through conversion functions when going between the VM and the Unsafe API.

// The conversion functions just happen to be no-ops at present.

static inline jlong field_offset_to_byte_offset(jlong field_offset) {

  return field_offset;

}

static inline jlong field_offset_from_byte_offset(jlong byte_offset) {

  return byte_offset;

}

static inline void assert_field_offset_sane(oop p, jlong field_offset) {

#ifdef ASSERT

  jlong byte_offset = field_offset_to_byte_offset(field_offset);

  if (p != NULL) {

    assert(byte_offset >= 0 && byte_offset <= (jlong)MAX_OBJECT_SIZE, "sane offset");

    if (byte_offset == (jint)byte_offset) {

      void* ptr_plus_disp = (address)p + byte_offset;

      assert(p->field_addr_raw((jint)byte_offset) == ptr_plus_disp,

             "raw [ptr+disp] must be consistent with oop::field_addr_raw");

    }

    jlong p_size = HeapWordSize * (jlong)(p->size());

    assert(byte_offset < p_size, "Unsafe access: offset " INT64_FORMAT " > object's size " INT64_FORMAT, (int64_t)byte_offset, (int64_t)p_size);

  }

#endif

}

static inline void* index_oop_from_field_offset_long(oop p, jlong field_offset) {

  assert_field_offset_sane(p, field_offset);

  jlong byte_offset = field_offset_to_byte_offset(field_offset);

  if (p != NULL) {

    p = Access<>::resolve(p);

  }

  if (sizeof(char*) == sizeof(jint)) {   // (this constant folds!)

    return (address)p + (jint) byte_offset;

  } else {

    return (address)p +        byte_offset;

  }

}

// Externally callable versions:

// (Use these in compiler intrinsics which emulate unsafe primitives.)

jlong Unsafe_field_offset_to_byte_offset(jlong field_offset) {

  return field_offset;

}

jlong Unsafe_field_offset_from_byte_offset(jlong byte_offset) {

  return byte_offset;

}

///// Data read/writes on the Java heap and in native (off-heap) memory

/**

 * Helper class for accessing memory.

 *

 * Normalizes values and wraps accesses in

 * JavaThread::doing_unsafe_access() if needed.

 */

template <typename T>

class MemoryAccess : StackObj {

  JavaThread* _thread;

  oop _obj;

  ptrdiff_t _offset;

  // Resolves and returns the address of the memory access.

  // This raw memory access may fault, so we make sure it happens within the

  // guarded scope by making the access volatile at least. Since the store

  // of Thread::set_doing_unsafe_access() is also volatile, these accesses

  // can not be reordered by the compiler. Therefore, if the access triggers

  // a fault, we will know that Thread::doing_unsafe_access() returns true.

  volatile T* addr() {

    void* addr = index_oop_from_field_offset_long(_obj, _offset);

    return static_cast<volatile T*>(addr);

  }

  template <typename U>

  U normalize_for_write(U x) {

    return x;

  }

  jboolean normalize_for_write(jboolean x) {

    return x & 1;

  }

  template <typename U>

  U normalize_for_read(U x) {

    return x;

  }

  jboolean normalize_for_read(jboolean x) {

    return x != 0;

  }

  /**

   * Helper class to wrap memory accesses in JavaThread::doing_unsafe_access()

   */

  class GuardUnsafeAccess {

    JavaThread* _thread;

  public:

    GuardUnsafeAccess(JavaThread* thread) : _thread(thread) {

      // native/off-heap access which may raise SIGBUS if accessing

      // memory mapped file data in a region of the file which has

      // been truncated and is now invalid

      _thread->set_doing_unsafe_access(true);

    }

    ~GuardUnsafeAccess() {

      _thread->set_doing_unsafe_access(false);

    }

  };

public:

  MemoryAccess(JavaThread* thread, jobject obj, jlong offset)

    : _thread(thread), _obj(JNIHandles::resolve(obj)), _offset((ptrdiff_t)offset) {

    assert_field_offset_sane(_obj, offset);

  }

  T get() {

    if (_obj == NULL) {

      GuardUnsafeAccess guard(_thread);

      T ret = RawAccess<>::load(addr());

      return normalize_for_read(ret);

    } else {

      T ret = HeapAccess<>::load_at(_obj, _offset);

      return normalize_for_read(ret);

    }

  }

  void put(T x) {

    if (_obj == NULL) {

      GuardUnsafeAccess guard(_thread);

      RawAccess<>::store(addr(), normalize_for_write(x));

    } else {

      HeapAccess<>::store_at(_obj, _offset, normalize_for_write(x));

    }

  }

  T get_volatile() {

    if (_obj == NULL) {

      GuardUnsafeAccess guard(_thread);

      volatile T ret = RawAccess<MO_SEQ_CST>::load(addr());

      return normalize_for_read(ret);

    } else {

      T ret = HeapAccess<MO_SEQ_CST>::load_at(_obj, _offset);

      return normalize_for_read(ret);

    }

  }

  void put_volatile(T x) {

    if (_obj == NULL) {

      GuardUnsafeAccess guard(_thread);

      RawAccess<MO_SEQ_CST>::store(addr(), normalize_for_write(x));

    } else {

      HeapAccess<MO_SEQ_CST>::store_at(_obj, _offset, normalize_for_write(x));

    }

  }

};

// These functions allow a null base pointer with an arbitrary address.

// But if the base pointer is non-null, the offset should make some sense.

// That is, it should be in the range [0, MAX_OBJECT_SIZE].

UNSAFE_ENTRY(jobject, Unsafe_GetReference(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) {

  oop p = JNIHandles::resolve(obj);

  assert_field_offset_sane(p, offset);

  oop v = HeapAccess<ON_UNKNOWN_OOP_REF>::oop_load_at(p, offset);

  return JNIHandles::make_local(env, v);

} UNSAFE_END

UNSAFE_ENTRY(void, Unsafe_PutReference(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject x_h)) {

  oop x = JNIHandles::resolve(x_h);

  oop p = JNIHandles::resolve(obj);

  assert_field_offset_sane(p, offset);

  HeapAccess<ON_UNKNOWN_OOP_REF>::oop_store_at(p, offset, x);

} UNSAFE_END

UNSAFE_ENTRY(jobject, Unsafe_GetReferenceVolatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) {

  oop p = JNIHandles::resolve(obj);

  assert_field_offset_sane(p, offset);

  oop v = HeapAccess<MO_SEQ_CST | ON_UNKNOWN_OOP_REF>::oop_load_at(p, offset);

  return JNIHandles::make_local(env, v);

} UNSAFE_END

UNSAFE_ENTRY(void, Unsafe_PutReferenceVolatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject x_h)) {

  oop x = JNIHandles::resolve(x_h);

  oop p = JNIHandles::resolve(obj);

  assert_field_offset_sane(p, offset);

  HeapAccess<MO_SEQ_CST | ON_UNKNOWN_OOP_REF>::oop_store_at(p, offset, x);

} UNSAFE_END

UNSAFE_ENTRY(jobject, Unsafe_GetUncompressedObject(JNIEnv *env, jobject unsafe, jlong addr)) {

  oop v = *(oop*) (address) addr;

  return JNIHandles::make_local(env, v);

} UNSAFE_END

UNSAFE_LEAF(jboolean, Unsafe_isBigEndian0(JNIEnv *env, jobject unsafe)) {

#ifdef VM_LITTLE_ENDIAN

  return false;

#else

  return true;

#endif

} UNSAFE_END

UNSAFE_LEAF(jint, Unsafe_unalignedAccess0(JNIEnv *env, jobject unsafe)) {

  return UseUnalignedAccesses;

} UNSAFE_END

#define DEFINE_GETSETOOP(java_type, Type) \

 \

UNSAFE_ENTRY(java_type, Unsafe_Get##Type(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) { \

  return MemoryAccess<java_type>(thread, obj, offset).get(); \

} UNSAFE_END \

 \

UNSAFE_ENTRY(void, Unsafe_Put##Type(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, java_type x)) { \

  MemoryAccess<java_type>(thread, obj, offset).put(x); \

} UNSAFE_END \

 \

// END DEFINE_GETSETOOP.

DEFINE_GETSETOOP(jboolean, Boolean)

DEFINE_GETSETOOP(jbyte, Byte)

DEFINE_GETSETOOP(jshort, Short);

DEFINE_GETSETOOP(jchar, Char);

DEFINE_GETSETOOP(jint, Int);

DEFINE_GETSETOOP(jlong, Long);

DEFINE_GETSETOOP(jfloat, Float);

DEFINE_GETSETOOP(jdouble, Double);

#undef DEFINE_GETSETOOP

#define DEFINE_GETSETOOP_VOLATILE(java_type, Type) \

 \

UNSAFE_ENTRY(java_type, Unsafe_Get##Type##Volatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) { \

  return MemoryAccess<java_type>(thread, obj, offset).get_volatile(); \

} UNSAFE_END \

 \

UNSAFE_ENTRY(void, Unsafe_Put##Type##Volatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, java_type x)) { \

  MemoryAccess<java_type>(thread, obj, offset).put_volatile(x); \

} UNSAFE_END \

 \

// END DEFINE_GETSETOOP_VOLATILE.

DEFINE_GETSETOOP_VOLATILE(jboolean, Boolean)

DEFINE_GETSETOOP_VOLATILE(jbyte, Byte)

DEFINE_GETSETOOP_VOLATILE(jshort, Short);

DEFINE_GETSETOOP_VOLATILE(jchar, Char);

DEFINE_GETSETOOP_VOLATILE(jint, Int);

DEFINE_GETSETOOP_VOLATILE(jlong, Long);

DEFINE_GETSETOOP_VOLATILE(jfloat, Float);

DEFINE_GETSETOOP_VOLATILE(jdouble, Double);

#undef DEFINE_GETSETOOP_VOLATILE

UNSAFE_LEAF(void, Unsafe_LoadFence(JNIEnv *env, jobject unsafe)) {

  OrderAccess::acquire();

} UNSAFE_END

UNSAFE_LEAF(void, Unsafe_StoreFence(JNIEnv *env, jobject unsafe)) {

  OrderAccess::release();

} UNSAFE_END

UNSAFE_LEAF(void, Unsafe_FullFence(JNIEnv *env, jobject unsafe)) {

  OrderAccess::fence();

} UNSAFE_END

////// Allocation requests

UNSAFE_ENTRY(jobject, Unsafe_AllocateInstance(JNIEnv *env, jobject unsafe, jclass cls)) {

  ThreadToNativeFromVM ttnfv(thread);

  return env->AllocObject(cls);

} UNSAFE_END

UNSAFE_ENTRY(jlong, Unsafe_AllocateMemory0(JNIEnv *env, jobject unsafe, jlong size)) {

  size_t sz = (size_t)size;

  sz = align_up(sz, HeapWordSize);

  void* x = os::malloc(sz, mtOther);

  return addr_to_java(x);

} UNSAFE_END

UNSAFE_ENTRY(jlong, Unsafe_ReallocateMemory0(JNIEnv *env, jobject unsafe, jlong addr, jlong size)) {

  void* p = addr_from_java(addr);

  size_t sz = (size_t)size;

  sz = align_up(sz, HeapWordSize);

  void* x = os::realloc(p, sz, mtOther);

  return addr_to_java(x);

} UNSAFE_END

UNSAFE_ENTRY(void, Unsafe_FreeMemory0(JNIEnv *env, jobject unsafe, jlong addr)) {

  void* p = addr_from_java(addr);

  os::free(p);

} UNSAFE_END

UNSAFE_ENTRY(void, Unsafe_SetMemory0(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jlong size, jbyte value)) {

  size_t sz = (size_t)size;

  oop base = JNIHandles::resolve(obj);

  void* p = index_oop_from_field_offset_long(base, offset);

  Copy::fill_to_memory_atomic(p, sz, value);

} UNSAFE_END

UNSAFE_ENTRY(void, Unsafe_CopyMemory0(JNIEnv *env, jobject unsafe, jobject srcObj, jlong srcOffset, jobject dstObj, jlong dstOffset, jlong size)) {

  size_t sz = (size_t)size;

  oop srcp = JNIHandles::resolve(srcObj);

  oop dstp = JNIHandles::resolve(dstObj);

  void* src = index_oop_from_field_offset_long(srcp, srcOffset);

  void* dst = index_oop_from_field_offset_long(dstp, dstOffset);

  Copy::conjoint_memory_atomic(src, dst, sz);

} UNSAFE_END

// This function is a leaf since if the source and destination are both in native memory

// the copy may potentially be very large, and we don't want to disable GC if we can avoid it.

// If either source or destination (or both) are on the heap, the function will enter VM using

// JVM_ENTRY_FROM_LEAF

UNSAFE_LEAF(void, Unsafe_CopySwapMemory0(JNIEnv *env, jobject unsafe, jobject srcObj, jlong srcOffset, jobject dstObj, jlong dstOffset, jlong size, jlong elemSize)) {

  size_t sz = (size_t)size;

  size_t esz = (size_t)elemSize;

  if (srcObj == NULL && dstObj == NULL) {

    // Both src & dst are in native memory

    address src = (address)srcOffset;

    address dst = (address)dstOffset;

    Copy::conjoint_swap(src, dst, sz, esz);

  } else {

    // At least one of src/dst are on heap, transition to VM to access raw pointers

    JVM_ENTRY_FROM_LEAF(env, void, Unsafe_CopySwapMemory0) {

      oop srcp = JNIHandles::resolve(srcObj);

      oop dstp = JNIHandles::resolve(dstObj);

      address src = (address)index_oop_from_field_offset_long(srcp, srcOffset);

      address dst = (address)index_oop_from_field_offset_long(dstp, dstOffset);

      Copy::conjoint_swap(src, dst, sz, esz);

    } JVM_END

  }

} UNSAFE_END

////// Random queries

UNSAFE_LEAF(jint, Unsafe_AddressSize0(JNIEnv *env, jobject unsafe)) {

  return sizeof(void*);

} UNSAFE_END

UNSAFE_LEAF(jint, Unsafe_PageSize()) {

  return os::vm_page_size();

} UNSAFE_END

static jlong find_field_offset(jclass clazz, jstring name, TRAPS) {

  assert(clazz != NULL, "clazz must not be NULL");

  assert(name != NULL, "name must not be NULL");

  ResourceMark rm(THREAD);

  char *utf_name = java_lang_String::as_utf8_string(JNIHandles::resolve_non_null(name));

  InstanceKlass* k = InstanceKlass::cast(java_lang_Class::as_Klass(JNIHandles::resolve_non_null(clazz)));

  jint offset = -1;

  for (JavaFieldStream fs(k); !fs.done(); fs.next()) {

    Symbol *name = fs.name();

    if (name->equals(utf_name)) {

      offset = fs.offset();

      break;

    }

  }

  if (offset < 0) {

    THROW_0(vmSymbols::java_lang_InternalError());

  }

  return field_offset_from_byte_offset(offset);

}

static jlong find_field_offset(jobject field, int must_be_static, TRAPS) {

  assert(field != NULL, "field must not be NULL");

  oop reflected   = JNIHandles::resolve_non_null(field);

  oop mirror      = java_lang_reflect_Field::clazz(reflected);

  Klass* k        = java_lang_Class::as_Klass(mirror);

  int slot        = java_lang_reflect_Field::slot(reflected);

  int modifiers   = java_lang_reflect_Field::modifiers(reflected);

  if (must_be_static >= 0) {

    int really_is_static = ((modifiers & JVM_ACC_STATIC) != 0);

    if (must_be_static != really_is_static) {