--- a/src/hotspot/share/gc/z/zPageAllocator.cpp	Thu Apr 25 08:55:50 2019 +0200
+++ b/src/hotspot/share/gc/z/zPageAllocator.cpp	Thu Apr 25 08:55:50 2019 +0200
@@ -37,6 +37,7 @@
 #include "runtime/init.hpp"
 
 static const ZStatCounter       ZCounterAllocationRate("Memory", "Allocation Rate", ZStatUnitBytesPerSecond);
+static const ZStatCounter       ZCounterPageCacheEvict("Memory", "Page Cache Evict", ZStatUnitBytesPerSecond);
 static const ZStatCriticalPhase ZCriticalPhaseAllocationStall("Allocation Stall");
 
 class ZPageAllocRequest : public StackObj {
@@ -271,16 +272,6 @@
   pmem.clear();
 }
 
-void ZPageAllocator::flush_cache(size_t size) {
-  ZList<ZPage> list;
-
-  _cache.flush(&list, size);
-
-  for (ZPage* page = list.remove_first(); page != NULL; page = list.remove_first()) {
-    destroy_page(page);
-  }
-}
-
 void ZPageAllocator::check_out_of_memory_during_initialization() {
   if (!is_init_completed()) {
     vm_exit_during_initialization("java.lang.OutOfMemoryError", "Java heap too small");
@@ -313,8 +304,11 @@
   // Try ensure that physical memory is available
   const size_t unused = try_ensure_unused(size, flags.no_reserve());
   if (unused < size) {
-    // Flush cache to free up more physical memory
-    flush_cache(size - unused);
+    // Try evict pages from the cache
+    const size_t needed = size - unused;
+    if (_cache.available() >= needed) {
+      evict_cache(needed);
+    }
   }
 
   // Create new page and allocate physical memory
@@ -468,6 +462,61 @@
   satisfy_alloc_queue();
 }
 
+void ZPageAllocator::flush_cache(ZPageCacheFlushClosure* cl) {
+  ZList<ZPage> list;
+
+  _cache.flush(cl, &list);
+
+  for (ZPage* page = list.remove_first(); page != NULL; page = list.remove_first()) {
+    destroy_page(page);
+  }
+}
+
+class ZPageCacheEvictClosure : public ZPageCacheFlushClosure {
+private:
+  const size_t _requested;
+  size_t       _evicted;
+
+public:
+  ZPageCacheEvictClosure(size_t requested) :
+      _requested(requested),
+      _evicted(0) {}
+
+  virtual bool do_page(const ZPage* page) {
+    if (_evicted < _requested) {
+      // Evict page
+      _evicted += page->size();
+      return true;
+    }
+
+    // Don't evict page
+    return false;
+  }
+
+  size_t evicted() const {
+    return _evicted;
+  }
+};
+
+void ZPageAllocator::evict_cache(size_t requested) {
+  // Evict pages
+  ZPageCacheEvictClosure cl(requested);
+  flush_cache(&cl);
+
+  const size_t evicted = cl.evicted();
+  const size_t cached_after = _cache.available();
+  const size_t cached_before = cached_after + evicted;
+
+  log_info(gc, heap)("Page Cache: " SIZE_FORMAT "M(%.0lf%%)->" SIZE_FORMAT "M(%.0lf%%), "
+                     "Evicted: " SIZE_FORMAT "M, Requested: " SIZE_FORMAT "M",
+                     cached_before / M, percent_of(cached_before, max_capacity()),
+                     cached_after / M, percent_of(cached_after, max_capacity()),
+                     evicted / M, requested / M);
+
+  // Update statistics
+  ZStatInc(ZCounterPageCacheEvict, evicted);
+}
+
 void ZPageAllocator::enable_deferred_delete() const {
   _safe_delete.enable_deferred_delete();
 }

--- a/src/hotspot/share/gc/z/zPageAllocator.hpp	Thu Apr 25 08:55:50 2019 +0200
+++ b/src/hotspot/share/gc/z/zPageAllocator.hpp	Thu Apr 25 08:55:50 2019 +0200
@@ -67,7 +67,8 @@
   void destroy_page(ZPage* page);
 
   void flush_pre_mapped();
-  void flush_cache(size_t size);
+  void flush_cache(ZPageCacheFlushClosure* cl);
+  void evict_cache(size_t requested);
 
   void check_out_of_memory_during_initialization();
 

--- a/src/hotspot/share/gc/z/zPageCache.cpp	Thu Apr 25 08:55:50 2019 +0200
+++ b/src/hotspot/share/gc/z/zPageCache.cpp	Thu Apr 25 08:55:50 2019 +0200
@@ -32,7 +32,6 @@
 static const ZStatCounter ZCounterPageCacheHitL1("Memory", "Page Cache Hit L1", ZStatUnitOpsPerSecond);
 static const ZStatCounter ZCounterPageCacheHitL2("Memory", "Page Cache Hit L2", ZStatUnitOpsPerSecond);
 static const ZStatCounter ZCounterPageCacheMiss("Memory", "Page Cache Miss", ZStatUnitOpsPerSecond);
-static const ZStatCounter ZCounterPageCacheFlush("Memory", "Page Cache Flush", ZStatUnitBytesPerSecond);
 
 ZPageCache::ZPageCache() :
     _available(0),
@@ -130,64 +129,49 @@
   _available += page->size();
 }
 
-void ZPageCache::flush_list(ZList<ZPage>* from, size_t requested, ZList<ZPage>* to, size_t* flushed) {
-  while (*flushed < requested) {
-    // Flush least recently used
-    ZPage* const page = from->remove_last();
-    if (page == NULL) {
-      break;
-    }
+bool ZPageCache::flush_list_inner(ZPageCacheFlushClosure* cl, ZList<ZPage>* from, ZList<ZPage>* to) {
+  ZPage* const page = from->last();
+  if (page == NULL || !cl->do_page(page)) {
+    // Don't flush page
+    return false;
+  }
 
-    *flushed += page->size();
-    to->insert_last(page);
-  }
+  // Flush page
+  _available -= page->size();
+  from->remove(page);
+  to->insert_last(page);
+  return true;
 }
 
-void ZPageCache::flush_per_numa_lists(ZPerNUMA<ZList<ZPage> >* from, size_t requested, ZList<ZPage>* to, size_t* flushed) {
+void ZPageCache::flush_list(ZPageCacheFlushClosure* cl, ZList<ZPage>* from, ZList<ZPage>* to) {
+  while (flush_list_inner(cl, from, to));
+}
+
+void ZPageCache::flush_per_numa_lists(ZPageCacheFlushClosure* cl, ZPerNUMA<ZList<ZPage> >* from, ZList<ZPage>* to) {
   const uint32_t numa_count = ZNUMA::count();
-  uint32_t numa_empty = 0;
+  uint32_t numa_done = 0;
   uint32_t numa_next = 0;
 
   // Flush lists round-robin
-  while (*flushed < requested) {
-    ZPage* const page = from->get(numa_next).remove_last();
-
+  while (numa_done < numa_count) {
+    ZList<ZPage>* numa_list = from->addr(numa_next);
     if (++numa_next == numa_count) {
       numa_next = 0;
     }
 
-    if (page == NULL) {
-      // List is empty
-      if (++numa_empty == numa_count) {
-        // All lists are empty
-        break;
-      }
-
-      // Try next list
-      continue;
+    if (flush_list_inner(cl, numa_list, to)) {
+      // Not done
+      numa_done = 0;
+    } else {
+      // Done
+      numa_done++;
     }
-
-    // Flush page
-    numa_empty = 0;
-    *flushed += page->size();
-    to->insert_last(page);
   }
 }
 
-void ZPageCache::flush(ZList<ZPage>* to, size_t requested) {
-  size_t flushed = 0;
-
+void ZPageCache::flush(ZPageCacheFlushClosure* cl, ZList<ZPage>* to) {
   // Prefer flushing large, then medium and last small pages
-  flush_list(&_large, requested, to, &flushed);
-  flush_list(&_medium, requested, to, &flushed);
-  flush_per_numa_lists(&_small, requested, to, &flushed);
-
-  ZStatInc(ZCounterPageCacheFlush, flushed);
-
-  log_info(gc, heap)("Page Cache Flushed: "
-                     SIZE_FORMAT "M requested, "
-                     SIZE_FORMAT "M(" SIZE_FORMAT "M->" SIZE_FORMAT "M) flushed",
-                     requested / M, flushed / M , _available / M, (_available - flushed) / M);
-
-  _available -= flushed;
+  flush_list(cl, &_large, to);
+  flush_list(cl, &_medium, to);
+  flush_per_numa_lists(cl, &_small, to);
 }

--- a/src/hotspot/share/gc/z/zPageCache.hpp	Thu Apr 25 08:55:50 2019 +0200
+++ b/src/hotspot/share/gc/z/zPageCache.hpp	Thu Apr 25 08:55:50 2019 +0200
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -29,10 +29,14 @@
 #include "gc/z/zValue.hpp"
 #include "memory/allocation.hpp"
 
+class ZPageCacheFlushClosure : public StackObj {
+public:
+  virtual bool do_page(const ZPage* page) = 0;
+};
+
 class ZPageCache {
 private:
   size_t                  _available;
-
   ZPerNUMA<ZList<ZPage> > _small;
   ZList<ZPage>            _medium;
   ZList<ZPage>            _large;
@@ -41,8 +45,9 @@
   ZPage* alloc_medium_page();
   ZPage* alloc_large_page(size_t size);
 
-  void flush_list(ZList<ZPage>* from, size_t requested, ZList<ZPage>* to, size_t* flushed);
-  void flush_per_numa_lists(ZPerNUMA<ZList<ZPage> >* from, size_t requested, ZList<ZPage>* to, size_t* flushed);
+  bool flush_list_inner(ZPageCacheFlushClosure* cl, ZList<ZPage>* from, ZList<ZPage>* to);
+  void flush_list(ZPageCacheFlushClosure* cl, ZList<ZPage>* from, ZList<ZPage>* to);
+  void flush_per_numa_lists(ZPageCacheFlushClosure* cl, ZPerNUMA<ZList<ZPage> >* from, ZList<ZPage>* to);
 
 public:
   ZPageCache();
@@ -52,7 +57,7 @@
   ZPage* alloc_page(uint8_t type, size_t size);
   void free_page(ZPage* page);
 
-  void flush(ZList<ZPage>* to, size_t requested);
+  void flush(ZPageCacheFlushClosure* cl, ZList<ZPage>* to);
 };
 
 #endif // SHARE_GC_Z_ZPAGECACHE_HPP