# HG changeset patch
# User ccheung
# Date 1522385318 25200
# Node ID a74836b05c2856d3192fa1c1f53cf3b03467d21a
# Parent  1e11c8a238f7693f50a56b0b9a9875bd7d592524
8200078: [Graal] runtime/appcds/GraalWithLimitedMetaspace.java crashes in visit_all_interfaces
Summary: stop CDS dumping right away when an OOM due to insufficient metaspace is encountered
Reviewed-by: iklam, mseledtsov

diff -r 1e11c8a238f7 -r a74836b05c28 src/hotspot/share/memory/metaspace.cpp
--- a/src/hotspot/share/memory/metaspace.cpp	Thu Mar 29 20:15:23 2018 -0700
+++ b/src/hotspot/share/memory/metaspace.cpp	Thu Mar 29 21:48:38 2018 -0700
@@ -4663,17 +4663,13 @@
   MetaWord* result = loader_data->metaspace_non_null()->allocate(word_size, mdtype);
 
   if (result == NULL) {
-    if (DumpSharedSpaces && THREAD->is_VM_thread()) {
-      tty->print_cr("Failed allocating metaspace object type %s of size " SIZE_FORMAT ". CDS dump aborted.",
-          MetaspaceObj::type_name(type), word_size * BytesPerWord);
-      vm_exit(1);
-    }
-
     tracer()->report_metaspace_allocation_failure(loader_data, word_size, type, mdtype);
 
     // Allocation failed.
-    if (is_init_completed()) {
+    if (is_init_completed() && !(DumpSharedSpaces && THREAD->is_VM_thread())) {
       // Only start a GC if the bootstrapping has completed.
+      // Also, we cannot GC if we are at the end of the CDS dumping stage which runs inside
+      // the VM thread.
 
       // Try to clean out some memory and retry.
       result = Universe::heap()->satisfy_failed_metadata_allocation(loader_data, word_size, mdtype);
@@ -4681,6 +4677,14 @@
   }
 
   if (result == NULL) {
+    if (DumpSharedSpaces) {
+      // CDS dumping keeps loading classes, so if we hit an OOM we probably will keep hitting OOM.
+      // We should abort to avoid generating a potentially bad archive.
+      tty->print_cr("Failed allocating metaspace object type %s of size " SIZE_FORMAT ". CDS dump aborted.",
+          MetaspaceObj::type_name(type), word_size * BytesPerWord);
+      tty->print_cr("Please increase MaxMetaspaceSize (currently " SIZE_FORMAT " bytes).", MaxMetaspaceSize);
+      vm_exit(1);
+    }
     report_metadata_oome(loader_data, word_size, type, mdtype, CHECK_NULL);
   }
 
diff -r 1e11c8a238f7 -r a74836b05c28 test/hotspot/jtreg/runtime/SharedArchiveFile/MaxMetaspaceSize.java
--- a/test/hotspot/jtreg/runtime/SharedArchiveFile/MaxMetaspaceSize.java	Thu Mar 29 20:15:23 2018 -0700
+++ b/test/hotspot/jtreg/runtime/SharedArchiveFile/MaxMetaspaceSize.java	Thu Mar 29 21:48:38 2018 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,7 +24,7 @@
 /**
  * @test
  * @requires vm.cds
- * @bug 8067187
+ * @bug 8067187 8200078
  * @summary Testing CDS dumping with the -XX:MaxMetaspaceSize=<size> option
  * @library /test/lib
  * @modules java.base/jdk.internal.misc
@@ -50,7 +50,7 @@
       processArgs.add("-XX:MaxMetaspaceSize=1m");
     }
 
-    String msg = "OutOfMemoryError: Metaspace";
+    String msg = "Failed allocating metaspace object";
     ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(processArgs.toArray(new String[0]));
     CDSTestUtils.executeAndLog(pb, "dump").shouldContain(msg).shouldHaveExitValue(1);
   }
diff -r 1e11c8a238f7 -r a74836b05c28 test/hotspot/jtreg/runtime/appcds/GraalWithLimitedMetaspace.java
--- a/test/hotspot/jtreg/runtime/appcds/GraalWithLimitedMetaspace.java	Thu Mar 29 20:15:23 2018 -0700
+++ b/test/hotspot/jtreg/runtime/appcds/GraalWithLimitedMetaspace.java	Thu Mar 29 21:48:38 2018 -0700
@@ -125,8 +125,14 @@
             "-XX:MetaspaceSize=12M",
             "-XX:MaxMetaspaceSize=12M"));
 
-        OutputAnalyzer output = TestCommon.executeAndLog(pb, "dump-archive")
-            .shouldHaveExitValue(1)
-            .shouldContain("Failed allocating metaspace object type");
+        OutputAnalyzer output = TestCommon.executeAndLog(pb, "dump-archive");
+        int exitValue = output.getExitValue();
+        if (exitValue == 1) {
+            output.shouldContain("Failed allocating metaspace object type");
+        } else if (exitValue == 0) {
+            output.shouldContain("Loading classes to share");
+        } else {
+            throw new RuntimeException("Unexpected exit value " + exitValue);
+        }
     }
 }