--- a/make/test/JtregNativeHotspot.gmk	Tue Nov 28 17:11:02 2017 -0500
+++ b/make/test/JtregNativeHotspot.gmk	Wed Nov 29 09:26:58 2017 +0900
@@ -79,6 +79,7 @@
     $(TOPDIR)/test/hotspot/jtreg/serviceability/jvmti/ModuleAwareAgents/ClassLoadPrepare \
     $(TOPDIR)/test/hotspot/jtreg/serviceability/jvmti/ModuleAwareAgents/ThreadStart \
     $(TOPDIR)/test/hotspot/jtreg/serviceability/jvmti/StartPhase/AllowedFunctions \
+    $(TOPDIR)/test/hotspot/jtreg/serviceability/dcmd/jvmti/AttachFailed \
     #
 
 # Add conditional directories here when needed.
@@ -110,6 +111,8 @@
     BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libAllowedFunctions := -lc
     BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libRedefineDoubleDelete := -lc
     BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libHandshakeTransitionTest := -lc
+    BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libHasNoEntryPoint := -lc
+    BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libReturnError := -lc
 endif
 
 ifeq ($(OPENJDK_TARGET_OS), linux)

--- a/src/hotspot/share/prims/jvmtiExport.cpp	Tue Nov 28 17:11:02 2017 -0500
+++ b/src/hotspot/share/prims/jvmtiExport.cpp	Wed Nov 29 09:26:58 2017 +0900
@@ -2578,7 +2578,7 @@
 
 jint JvmtiExport::load_agent_library(const char *agent, const char *absParam,
                                      const char *options, outputStream* st) {
-  char ebuf[1024];
+  char ebuf[1024] = {0};
   char buffer[JVM_MAXPATHLEN];
   void* library = NULL;
   jint result = JNI_ERR;
@@ -2628,6 +2628,8 @@
       if (!agent_lib->is_static_lib()) {
         os::dll_unload(library);
       }
+      st->print_cr("%s is not available in %s",
+                   on_attach_symbols[0], agent_lib->name());
       delete agent_lib;
     } else {
       // Invoke the Agent_OnAttach function
@@ -2654,9 +2656,14 @@
       }
 
       // Agent_OnAttach executed so completion status is JNI_OK
-      st->print_cr("%d", result);
+      st->print_cr("return code: %d", result);
       result = JNI_OK;
     }
+  } else {
+    st->print_cr("%s was not loaded.", agent);
+    if (*ebuf != '\0') {
+      st->print_cr("%s", ebuf);
+    }
   }
   return result;
 }

--- a/src/jdk.attach/share/classes/sun/tools/attach/HotSpotVirtualMachine.java	Tue Nov 28 17:11:02 2017 -0500
+++ b/src/jdk.attach/share/classes/sun/tools/attach/HotSpotVirtualMachine.java	Wed Nov 29 09:26:58 2017 +0900
@@ -86,18 +86,23 @@
     private void loadAgentLibrary(String agentLibrary, boolean isAbsolute, String options)
         throws AgentLoadException, AgentInitializationException, IOException
     {
+        String msgPrefix = "return code: ";
         InputStream in = execute("load",
                                  agentLibrary,
                                  isAbsolute ? "true" : "false",
                                  options);
-        try {
-            int result = readInt(in);
-            if (result != 0) {
-                throw new AgentInitializationException("Agent_OnAttach failed", result);
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(in))) {
+            String result = reader.readLine();
+            if (result == null) {
+                throw new AgentLoadException("Target VM did not respond");
+            } else if (result.startsWith(msgPrefix)) {
+                int retCode = Integer.parseInt(result.substring(msgPrefix.length()));
+                if (retCode != 0) {
+                    throw new AgentInitializationException("Agent_OnAttach failed", retCode);
+                }
+            } else {
+                throw new AgentLoadException(result);
             }
-        } finally {
-            in.close();
-
         }
     }
 

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/jtreg/serviceability/dcmd/jvmti/AttachFailed/AttachFailedTestBase.java	Wed Nov 29 09:26:58 2017 +0900
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+import java.nio.file.Paths;
+import jdk.test.lib.dcmd.*;
+import jdk.test.lib.Platform;
+import org.testng.annotations.Test;
+
+public abstract class AttachFailedTestBase {
+
+    public abstract void run(CommandExecutor executor);
+
+    /**
+     * Build path to shared object according to platform rules
+     */
+    public static String getSharedObjectPath(String name) {
+        String libname;
+        if (Platform.isWindows()) {
+            libname = name + ".dll";
+        } else if (Platform.isOSX()) {
+            libname = "lib" + name + ".dylib";
+        } else {
+            libname = "lib" + name + ".so";
+        }
+
+        return Paths.get(System.getProperty("test.nativepath"), libname)
+                    .toAbsolutePath()
+                    .toString();
+    }
+
+    @Test
+    public void jmx() throws Throwable {
+        run(new JMXExecutor());
+    }
+
+    @Test
+    public void cli() throws Throwable {
+        run(new PidJcmdExecutor());
+    }
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/jtreg/serviceability/dcmd/jvmti/AttachFailed/AttachIncorrectLibrary.java	Wed Nov 29 09:26:58 2017 +0900
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+import jdk.test.lib.dcmd.*;
+import jdk.test.lib.process.OutputAnalyzer;
+
+/*
+ * @test
+ * @bug 8165736
+ * @library /test/lib
+ * @run testng AttachIncorrectLibrary
+ */
+public class AttachIncorrectLibrary extends AttachFailedTestBase {
+    @Override
+    public void run(CommandExecutor executor)  {
+        try {
+            OutputAnalyzer output = executor.execute("JVMTI.agent_load " +
+                                           getSharedObjectPath("SilverBullet"));
+            output.shouldContain(" was not loaded");
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/jtreg/serviceability/dcmd/jvmti/AttachFailed/AttachNoEntry.java	Wed Nov 29 09:26:58 2017 +0900
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+import jdk.test.lib.dcmd.*;
+import jdk.test.lib.process.OutputAnalyzer;
+
+/*
+ * @test
+ * @bug 8165736
+ * @library /test/lib
+ * @run testng AttachNoEntry
+ */
+public class AttachNoEntry extends AttachFailedTestBase {
+    @Override
+    public void run(CommandExecutor executor)  {
+        try {
+            String libpath = getSharedObjectPath("HasNoEntryPoint");
+            OutputAnalyzer output = null;
+
+            output = executor.execute("JVMTI.agent_load " + libpath);
+            output.shouldContain("Agent_OnAttach is not available");
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/jtreg/serviceability/dcmd/jvmti/AttachFailed/AttachReturnError.java	Wed Nov 29 09:26:58 2017 +0900
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+import jdk.test.lib.dcmd.*;
+import jdk.test.lib.process.OutputAnalyzer;
+
+/*
+ * @test
+ * @bug 8165736
+ * @library /test/lib
+ * @run testng AttachReturnError
+ */
+public class AttachReturnError extends AttachFailedTestBase {
+    @Override
+    public void run(CommandExecutor executor)  {
+        try {
+            String libpath = getSharedObjectPath("ReturnError");
+            OutputAnalyzer output = null;
+
+            output = executor.execute("JVMTI.agent_load " + libpath);
+            output.shouldContain("return code: -1");
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/jtreg/serviceability/dcmd/jvmti/AttachFailed/libHasNoEntryPoint.c	Wed Nov 29 09:26:58 2017 +0900
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+extern int dummy() {
+  return 0;
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/jtreg/serviceability/dcmd/jvmti/AttachFailed/libReturnError.c	Wed Nov 29 09:26:58 2017 +0900
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <jni.h>
+#include <jvmti.h>
+
+JNIEXPORT
+jint JNICALL Agent_OnAttach(JavaVM *vm, char *options, void *reserved) {
+  return JNI_ERR;
+}

--- a/test/jdk/com/sun/tools/attach/StartManagementAgent.java	Tue Nov 28 17:11:02 2017 -0500
+++ b/test/jdk/com/sun/tools/attach/StartManagementAgent.java	Wed Nov 29 09:26:58 2017 +0900
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -99,7 +99,7 @@
         } catch(AttachOperationFailedException ex) {
             // We expect parsing of "apa" above to fail, but if the file path
             // can't be read we get a different exception message
-            if (!ex.getMessage().contains("Invalid com.sun.management.jmxremote.port number")) {
+            if (!ex.getMessage().contains("NumberFormatException: For input string: \"apa\"")) {
                 throw ex;
             }
             ex.printStackTrace(System.err);