8201375: Add the AllowArchivingWithJavaAgent diagnostic vm option to allow the use of the -javaagent option during CDS dumping
authorccheung
Fri, 16 Nov 2018 16:10:25 -0800
changeset 52596 dfa02b3f728c
parent 52595 16609197022c
child 52597 3cda8fed1524
8201375: Add the AllowArchivingWithJavaAgent diagnostic vm option to allow the use of the -javaagent option during CDS dumping Summary: Allow Java agent during CDS dumping if the AllowArchivingWithJavaAgent dignostic option is specified. Reviewed-by: iklam, jiangli, sspitsyn, dcubed
src/hotspot/share/memory/filemap.cpp
src/hotspot/share/memory/filemap.hpp
src/hotspot/share/memory/metaspaceShared.cpp
src/hotspot/share/runtime/globals.hpp
src/hotspot/share/runtime/thread.cpp
test/hotspot/jtreg/runtime/appcds/javaldr/AnonVmClassesDuringDump.java
test/hotspot/jtreg/runtime/appcds/javaldr/GCDuringDump.java
test/hotspot/jtreg/runtime/appcds/javaldr/GCSharedStringsDuringDump.java
test/hotspot/jtreg/runtime/appcds/jvmti/dumpingWithAgent/DumpingWithJavaAgent.java
test/hotspot/jtreg/runtime/appcds/jvmti/dumpingWithAgent/DumpingWithJvmtiAgent.java
test/hotspot/jtreg/runtime/appcds/jvmti/dumpingWithAgent/SimpleAgent.java
test/hotspot/jtreg/runtime/appcds/jvmti/dumpingWithAgent/SimpleAgent.mf
test/hotspot/jtreg/runtime/appcds/jvmti/dumpingWithAgent/libAddToSystemCLSearchOnLoad.c
--- a/src/hotspot/share/memory/filemap.cpp	Fri Nov 16 23:39:51 2018 +0100
+++ b/src/hotspot/share/memory/filemap.cpp	Fri Nov 16 16:10:25 2018 -0800
@@ -210,6 +210,7 @@
   _verify_remote = BytecodeVerificationRemote;
   _has_platform_or_app_classes = ClassLoaderExt::has_platform_or_app_classes();
   _shared_base_address = SharedBaseAddress;
+  _allow_archiving_with_java_agent = AllowArchivingWithJavaAgent;
 }
 
 void SharedClassPathEntry::init(const char* name, bool is_modules_image, TRAPS) {
@@ -1358,6 +1359,21 @@
     return false;
   }
 
+  // Java agents are allowed during run time. Therefore, the following condition is not
+  // checked: (!_allow_archiving_with_java_agent && AllowArchivingWithJavaAgent)
+  // Note: _allow_archiving_with_java_agent is set in the shared archive during dump time
+  // while AllowArchivingWithJavaAgent is set during the current run.
+  if (_allow_archiving_with_java_agent && !AllowArchivingWithJavaAgent) {
+    FileMapInfo::fail_continue("The setting of the AllowArchivingWithJavaAgent is different "
+                               "from the setting in the shared archive.");
+    return false;
+  }
+
+  if (_allow_archiving_with_java_agent) {
+    warning("This archive was created with AllowArchivingWithJavaAgent. It should be used "
+            "for testing purposes only and should not be used in a production environment");
+  }
+
   return true;
 }
 
--- a/src/hotspot/share/memory/filemap.hpp	Fri Nov 16 23:39:51 2018 +0100
+++ b/src/hotspot/share/memory/filemap.hpp	Fri Nov 16 16:10:25 2018 -0800
@@ -151,6 +151,7 @@
   bool   _verify_remote;                // BytecodeVerificationRemote setting
   bool   _has_platform_or_app_classes;  // Archive contains app classes
   size_t _shared_base_address;          // SharedBaseAddress used at dump time
+  bool   _allow_archiving_with_java_agent; // setting of the AllowArchivingWithJavaAgent option
 
   void set_has_platform_or_app_classes(bool v) {
     _has_platform_or_app_classes = v;
--- a/src/hotspot/share/memory/metaspaceShared.cpp	Fri Nov 16 23:39:51 2018 +0100
+++ b/src/hotspot/share/memory/metaspaceShared.cpp	Fri Nov 16 16:10:25 2018 -0800
@@ -1494,6 +1494,12 @@
   if (PrintSystemDictionaryAtExit) {
     SystemDictionary::print();
   }
+
+  if (AllowArchivingWithJavaAgent) {
+    warning("This archive was created with AllowArchivingWithJavaAgent. It should be used "
+            "for testing purposes only and should not be used in a production environment");
+  }
+
   // There may be other pending VM operations that operate on the InstanceKlasses,
   // which will fail because InstanceKlasses::remove_unshareable_info()
   // has been called. Forget these operations and exit the VM directly.
--- a/src/hotspot/share/runtime/globals.hpp	Fri Nov 16 23:39:51 2018 +0100
+++ b/src/hotspot/share/runtime/globals.hpp	Fri Nov 16 16:10:25 2018 -0800
@@ -2431,6 +2431,9 @@
           "Average number of symbols per bucket in shared table")           \
           range(2, 246)                                                     \
                                                                             \
+  diagnostic(bool, AllowArchivingWithJavaAgent, false,                      \
+          "Allow Java agent to be run with CDS dumping")                    \
+                                                                            \
   diagnostic(bool, PrintMethodHandleStubs, false,                           \
           "Print generated stub code for method handles")                   \
                                                                             \
--- a/src/hotspot/share/runtime/thread.cpp	Fri Nov 16 23:39:51 2018 +0100
+++ b/src/hotspot/share/runtime/thread.cpp	Fri Nov 16 16:10:25 2018 -0800
@@ -4088,6 +4088,17 @@
   JvmtiExport::enter_onload_phase();
 
   for (agent = Arguments::agents(); agent != NULL; agent = agent->next()) {
+    // CDS dumping does not support native JVMTI agent.
+    // CDS dumping supports Java agent if the AllowArchivingWithJavaAgent diagnostic option is specified.
+    if (DumpSharedSpaces) {
+      if(!agent->is_instrument_lib()) {
+        vm_exit_during_cds_dumping("CDS dumping does not support native JVMTI agent, name", agent->name());
+      } else if (!AllowArchivingWithJavaAgent) {
+        vm_exit_during_cds_dumping(
+          "Must enable AllowArchivingWithJavaAgent in order to run Java agent during CDS dumping");
+      }
+    }
+
     OnLoadEntry_t  on_load_entry = lookup_agent_on_load(agent);
 
     if (on_load_entry != NULL) {
@@ -4100,6 +4111,7 @@
       vm_exit_during_initialization("Could not find Agent_OnLoad function in the agent library", agent->name());
     }
   }
+
   JvmtiExport::enter_primordial_phase();
 }
 
--- a/test/hotspot/jtreg/runtime/appcds/javaldr/AnonVmClassesDuringDump.java	Fri Nov 16 23:39:51 2018 +0100
+++ b/test/hotspot/jtreg/runtime/appcds/javaldr/AnonVmClassesDuringDump.java	Fri Nov 16 16:10:25 2018 -0800
@@ -47,6 +47,8 @@
         "AnonVmClassesDuringDumpTransformer",
     };
 
+    public static String cdsDiagnosticOption = "-XX:+AllowArchivingWithJavaAgent";
+
     public static void main(String[] args) throws Throwable {
         String agentJar =
             ClassFileInstaller.writeJar("AnonVmClassesDuringDumpTransformer.jar",
@@ -58,6 +60,7 @@
 
         TestCommon.testDump(appJar, TestCommon.list("Hello"),
                             "-javaagent:" + agentJar,
+                            "-XX:+UnlockDiagnosticVMOptions", cdsDiagnosticOption,
                             // Set the following property to see logs for dynamically generated classes
                             // in STDOUT
                             "-Djava.lang.invoke.MethodHandle.DUMP_CLASS_FILES=true");
@@ -70,11 +73,13 @@
         String suffix = ".*source: shared objects file.*";
         String pattern = prefix + class_pattern + suffix;
         // during run time, anonymous classes shouldn't be loaded from the archive
-        TestCommon.run("-cp", appJar, "Hello")
+        TestCommon.run("-cp", appJar,
+            "-XX:+UnlockDiagnosticVMOptions", cdsDiagnosticOption, "Hello")
             .assertNormalExit(output -> output.shouldNotMatch(pattern));
 
         // inspect the archive and make sure no anonymous class is in there
         TestCommon.run("-cp", appJar,
+            "-XX:+UnlockDiagnosticVMOptions", cdsDiagnosticOption,
             "-XX:+PrintSharedArchiveAndExit", "-XX:+PrintSharedDictionary", "Hello")
             .assertNormalExit(output -> output.shouldNotMatch(class_pattern));
     }
--- a/test/hotspot/jtreg/runtime/appcds/javaldr/GCDuringDump.java	Fri Nov 16 23:39:51 2018 +0100
+++ b/test/hotspot/jtreg/runtime/appcds/javaldr/GCDuringDump.java	Fri Nov 16 16:10:25 2018 -0800
@@ -64,14 +64,17 @@
             // i = 1 -- run with agent = cause extra GCs
 
             String extraArg = (i == 0) ? "-showversion" : "-javaagent:" + agentJar;
+            String extraOption = (i == 0) ? "-showversion" : "-XX:+AllowArchivingWithJavaAgent";
 
             TestCommon.testDump(appJar, TestCommon.list("Hello"),
+                                "-XX:+UnlockDiagnosticVMOptions", extraOption,
                                 extraArg, "-Xmx32m", gcLog);
 
             TestCommon.run(
                 "-cp", appJar,
                 "-Xmx32m",
                 "-XX:+PrintSharedSpaces",
+                "-XX:+UnlockDiagnosticVMOptions", extraOption,
                 gcLog,
                 "Hello")
               .assertNormalExit();
--- a/test/hotspot/jtreg/runtime/appcds/javaldr/GCSharedStringsDuringDump.java	Fri Nov 16 23:39:51 2018 +0100
+++ b/test/hotspot/jtreg/runtime/appcds/javaldr/GCSharedStringsDuringDump.java	Fri Nov 16 16:10:25 2018 -0800
@@ -88,11 +88,12 @@
             // i = 1 -- run with agent = cause extra GCs
 
             String extraArg = (i == 0) ? "-showversion" : "-javaagent:" + agentJar;
-
+            String extraOption = (i == 0) ? "-showversion" : "-XX:+AllowArchivingWithJavaAgent";
             OutputAnalyzer output = TestCommon.dump(
                                 appJar, TestCommon.list("GCSharedStringsDuringDumpWb"),
                                 bootClassPath, extraArg, "-Xmx32m", gcLog,
-                                "-XX:SharedArchiveConfigFile=" + sharedArchiveCfgFile);
+                                "-XX:SharedArchiveConfigFile=" + sharedArchiveCfgFile,
+                                "-XX:+UnlockDiagnosticVMOptions", extraOption);
 
             if (output.getStdout().contains("Too many string space regions") ||
                 output.getStderr().contains("Unable to write archive heap memory regions") ||
@@ -104,15 +105,19 @@
                 TestCommon.testDump(
                     appJar, TestCommon.list("GCSharedStringsDuringDumpWb"),
                     bootClassPath, extraArg, "-Xmx8g", "-XX:NewSize=8m", gcLog,
-                    "-XX:SharedArchiveConfigFile=" + sharedArchiveCfgFile);
+                    "-XX:SharedArchiveConfigFile=" + sharedArchiveCfgFile,
+                    "-XX:+UnlockDiagnosticVMOptions", extraOption);
             }
 
             TestCommon.run(
                 "-cp", appJar,
                 bootClassPath,
+                extraArg,
+                "-Xlog:cds=info,class+path=info",
                 "-Xmx32m",
                 "-XX:+PrintSharedSpaces",
                 "-XX:+UnlockDiagnosticVMOptions",
+                extraOption,
                 "-XX:+WhiteBoxAPI",
                 "-XX:SharedReadOnlySize=30m",
                 gcLog,
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/jtreg/runtime/appcds/jvmti/dumpingWithAgent/DumpingWithJavaAgent.java	Fri Nov 16 16:10:25 2018 -0800
@@ -0,0 +1,119 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/*
+ * @test
+ * @summary CDS dumping with java agent.
+ * @library /test/lib /test/hotspot/jtreg/runtime/appcds /test/hotspot/jtreg/runtime/appcds/test-classes
+ * @requires vm.cds
+ * @requires vm.flavor != "minimal"
+ * @modules jdk.jartool/sun.tools.jar
+ * @build SimpleAgent Hello
+ * @run main/othervm DumpingWithJavaAgent
+ */
+
+import jdk.test.lib.cds.CDSOptions;
+import jdk.test.lib.process.OutputAnalyzer;
+import jdk.test.lib.process.ProcessTools;
+
+public class DumpingWithJavaAgent {
+    public static String appClasses[] = {
+        "Hello",
+    };
+    public static String agentClasses[] = {
+        "SimpleAgent",
+    };
+
+    public static String warningMessages[] = {
+        "This archive was created with AllowArchivingWithJavaAgent",
+        "It should be used for testing purposes only and should not be used in a production environment",
+    };
+
+    public static String errorMessage =
+        "The setting of the AllowArchivingWithJavaAgent is different from the setting in the shared archive.";
+
+
+    public static String diagnosticOption = "-XX:+AllowArchivingWithJavaAgent";
+
+    public static void main(String[] args) throws Throwable {
+        String agentJar =
+            ClassFileInstaller.writeJar("SimpleAgent.jar",
+                                        ClassFileInstaller.Manifest.fromSourceFile("SimpleAgent.mf"),
+                                        agentClasses);
+
+        String appJar =
+            ClassFileInstaller.writeJar("DumpingWithJavaAgent.jar", appClasses);
+
+        // CDS dumping with a java agent with the AllowArchvingWithJavaAgent diagnostic option.
+        OutputAnalyzer output = TestCommon.testDump(appJar, TestCommon.list("Hello"),
+            "-XX:+UnlockDiagnosticVMOptions", diagnosticOption,
+            "-javaagent:" + agentJar);
+        TestCommon.checkDump(output);
+        output.shouldContain(warningMessages[0]);
+        output.shouldContain(warningMessages[1]);
+        output.shouldContain("inside SimpleAgent");
+
+        // Using the archive with the AllowArchvingWithJavaAgent diagnostic option.
+        output = TestCommon.exec(
+            appJar,
+            "-Xlog:class+load=trace",
+            "-XX:+UnlockDiagnosticVMOptions", diagnosticOption,
+            "Hello");
+        if (!TestCommon.isUnableToMap(output)) {
+            output.shouldHaveExitValue(0);
+            output.shouldContain(warningMessages[0]);
+            output.shouldContain(warningMessages[1]);
+            output.shouldContain("[class,load] Hello source: shared objects file");
+        }
+
+        // Using the archive with -Xshare:on without the diagnostic option.
+        // VM should exit with an error message.
+        output = TestCommon.exec(
+            appJar,
+            "Hello");
+        output.shouldHaveExitValue(1);
+        output.shouldContain(errorMessage);
+
+        // Using the archive with -Xshare:auto without the diagnostic option.
+        // VM should continue execution with a warning message. The archive
+        // will not be used.
+        output = TestCommon.execAuto(
+            "-cp", appJar,
+            "-Xlog:class+load=trace,cds=info",
+            "Hello");
+        if (!TestCommon.isUnableToMap(output)) {
+            output.shouldHaveExitValue(0);
+            output.shouldContain(errorMessage);
+            output.shouldMatch(".class.load. Hello source:.*DumpingWithJavaAgent.jar");
+
+        // CDS dumping with a java agent without the AllowArchvingWithJavaAgent diagnostic option.
+        // VM will exit with an error message.
+        output = TestCommon.dump(appJar, TestCommon.list("Hello"),
+            "-javaagent:" + agentJar);
+        }
+        output.shouldContain("Must enable AllowArchivingWithJavaAgent in order to run Java agent during CDS dumping")
+            .shouldHaveExitValue(1);
+    }
+}
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/jtreg/runtime/appcds/jvmti/dumpingWithAgent/DumpingWithJvmtiAgent.java	Fri Nov 16 16:10:25 2018 -0800
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @summary CDS dumping with JVMTI agent.
+ * @requires vm.cds
+ * @requires vm.flavor != "minimal"
+ * @library /test/lib /test/hotspot/jtreg/runtime/appcds
+ * @modules jdk.jartool/sun.tools.jar
+ * @compile ../../test-classes/Hello.java
+ * @run main/othervm/native DumpingWithJvmtiAgent
+ */
+
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import jdk.test.lib.process.OutputAnalyzer;
+
+public class DumpingWithJvmtiAgent {
+    private static final String AGENT_LIB_ONLOAD = "AddToSystemCLSearchOnLoad";
+
+    public static void main(String[] args) throws Exception {
+        String appJar = JarBuilder.getOrCreateHelloJar();
+
+        // CDS dump with a JVMTI agent with the AllowArchivingWithJavaAgent option.
+        // vm should exit with an error message.
+        OutputAnalyzer out = TestCommon.dump(
+           appJar,
+           TestCommon.list("Hello"),
+           "-XX:+UnlockDiagnosticVMOptions", "-XX:+AllowArchivingWithJavaAgent",
+           "-agentlib:" + AGENT_LIB_ONLOAD + "=" + appJar,
+           "-Djava.library.path=" + System.getProperty("java.library.path"));
+        out.shouldContain("CDS dumping does not support native JVMTI agent, name: " + AGENT_LIB_ONLOAD)
+           .shouldHaveExitValue(1);
+
+        // CDS dump with a JVMTI agent without the AllowArchivingWithJavaAgent option.
+        // vm should exit with an error message.
+        out = TestCommon.dump(
+           appJar,
+           TestCommon.list("Hello"),
+           "-agentlib:" + AGENT_LIB_ONLOAD + "=" + appJar,
+           "-Djava.library.path=" + System.getProperty("java.library.path"));
+        out.shouldContain("CDS dumping does not support native JVMTI agent, name: " + AGENT_LIB_ONLOAD)
+           .shouldHaveExitValue(1);
+    }
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/jtreg/runtime/appcds/jvmti/dumpingWithAgent/SimpleAgent.java	Fri Nov 16 16:10:25 2018 -0800
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+import java.lang.instrument.Instrumentation;
+
+public class SimpleAgent {
+    public static void premain(String agentArg, Instrumentation instrumentation) {
+        System.out.println("inside SimpleAgent");
+    }
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/jtreg/runtime/appcds/jvmti/dumpingWithAgent/SimpleAgent.mf	Fri Nov 16 16:10:25 2018 -0800
@@ -0,0 +1,2 @@
+Manifest-Version: 1.0
+Premain-Class: SimpleAgent
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/jtreg/runtime/appcds/jvmti/dumpingWithAgent/libAddToSystemCLSearchOnLoad.c	Fri Nov 16 16:10:25 2018 -0800
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <jvmti.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+    static jvmtiEnv *jvmti = NULL;
+
+    JNIEXPORT jint JNICALL Agent_OnLoad(JavaVM *jvm, char *options, void *reserved) {
+        int err = (*jvm)->GetEnv(jvm, (void**) &jvmti, JVMTI_VERSION_9);
+        if (err != JNI_OK) {
+            return JNI_ERR;
+        }
+        err = (*jvmti)->AddToSystemClassLoaderSearch(jvmti, (const char*)options);
+        if (err != JVMTI_ERROR_NONE) {
+            return JNI_ERR;
+        }
+        return JNI_OK;
+    }
+
+#ifdef __cplusplus
+}
+#endif