hotspot/src/share/vm/gc/parallel/mutableNUMASpace.cpp
author sangheki
Tue, 07 Mar 2017 10:25:58 -0800
changeset 46312 385a8b027e7d
parent 40655 9f644073d3a0
child 46620 750c6edff33b
permissions -rw-r--r--
8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled Summary: Exit VM when fails to initialize small initial heap with NUMA and large pages enabled Reviewed-by: jmasa, sjohanss
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
     1
/*
40655
9f644073d3a0 8157907: Incorrect inclusion of atomic.hpp instead of atomic.inline.hpp
dholmes
parents: 33105
diff changeset
     2
 * Copyright (c) 2006, 2016, Oracle and/or its affiliates. All rights reserved.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
     3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
489c9b5090e2 Initial load
duke
parents:
diff changeset
     4
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
     5
 * This code is free software; you can redistribute it and/or modify it
489c9b5090e2 Initial load
duke
parents:
diff changeset
     6
 * under the terms of the GNU General Public License version 2 only, as
489c9b5090e2 Initial load
duke
parents:
diff changeset
     7
 * published by the Free Software Foundation.
489c9b5090e2 Initial load
duke
parents:
diff changeset
     8
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
     9
 * This code is distributed in the hope that it will be useful, but WITHOUT
489c9b5090e2 Initial load
duke
parents:
diff changeset
    10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
489c9b5090e2 Initial load
duke
parents:
diff changeset
    11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
489c9b5090e2 Initial load
duke
parents:
diff changeset
    12
 * version 2 for more details (a copy is included in the LICENSE file that
489c9b5090e2 Initial load
duke
parents:
diff changeset
    13
 * accompanied this code).
489c9b5090e2 Initial load
duke
parents:
diff changeset
    14
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
    15
 * You should have received a copy of the GNU General Public License version
489c9b5090e2 Initial load
duke
parents:
diff changeset
    16
 * 2 along with this work; if not, write to the Free Software Foundation,
489c9b5090e2 Initial load
duke
parents:
diff changeset
    17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
489c9b5090e2 Initial load
duke
parents:
diff changeset
    18
 *
5547
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 2154
diff changeset
    19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 2154
diff changeset
    20
 * or visit www.oracle.com if you need additional information or have any
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 2154
diff changeset
    21
 * questions.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    22
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
    23
 */
489c9b5090e2 Initial load
duke
parents:
diff changeset
    24
7397
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 5702
diff changeset
    25
#include "precompiled.hpp"
30764
fec48bf5a827 8079792: GC directory structure cleanup
pliden
parents: 30154
diff changeset
    26
#include "gc/parallel/mutableNUMASpace.hpp"
fec48bf5a827 8079792: GC directory structure cleanup
pliden
parents: 30154
diff changeset
    27
#include "gc/shared/collectedHeap.hpp"
fec48bf5a827 8079792: GC directory structure cleanup
pliden
parents: 30154
diff changeset
    28
#include "gc/shared/spaceDecorator.hpp"
7397
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 5702
diff changeset
    29
#include "oops/oop.inline.hpp"
40655
9f644073d3a0 8157907: Incorrect inclusion of atomic.hpp instead of atomic.inline.hpp
dholmes
parents: 33105
diff changeset
    30
#include "runtime/atomic.hpp"
14583
d70ee55535f4 8003935: Simplify the needed includes for using Thread::current()
stefank
parents: 13195
diff changeset
    31
#include "runtime/thread.inline.hpp"
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    32
46312
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    33
MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment), _must_use_large_pages(false) {
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 12379
diff changeset
    34
  _lgrp_spaces = new (ResourceObj::C_HEAP, mtGC) GrowableArray<LGRPSpace*>(0, true);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    35
  _page_size = os::vm_page_size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
    36
  _adaptation_cycles = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    37
  _samples_count = 0;
46312
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    38
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    39
#ifdef LINUX
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    40
  // Changing the page size can lead to freeing of memory. When using large pages
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    41
  // and the memory has been both reserved and committed, Linux does not support
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    42
  // freeing parts of it.
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    43
    if (UseLargePages && !os::can_commit_large_page_memory()) {
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    44
      _must_use_large_pages = true;
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    45
    }
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    46
#endif // LINUX
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    47
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    48
  update_layout(true);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    49
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
    50
489c9b5090e2 Initial load
duke
parents:
diff changeset
    51
MutableNUMASpace::~MutableNUMASpace() {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    52
  for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    53
    delete lgrp_spaces()->at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    54
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
    55
  delete lgrp_spaces();
489c9b5090e2 Initial load
duke
parents:
diff changeset
    56
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
    57
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    58
#ifndef PRODUCT
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    59
void MutableNUMASpace::mangle_unused_area() {
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    60
  // This method should do nothing.
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    61
  // It can be called on a numa space during a full compaction.
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    62
}
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    63
void MutableNUMASpace::mangle_unused_area_complete() {
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    64
  // This method should do nothing.
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    65
  // It can be called on a numa space during a full compaction.
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    66
}
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    67
void MutableNUMASpace::mangle_region(MemRegion mr) {
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    68
  // This method should do nothing because numa spaces are not mangled.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    69
}
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    70
void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    71
  assert(false, "Do not mangle MutableNUMASpace's");
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    72
}
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    73
void MutableNUMASpace::set_top_for_allocations() {
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    74
  // This method should do nothing.
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    75
}
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    76
void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    77
  // This method should do nothing.
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    78
}
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    79
void MutableNUMASpace::check_mangled_unused_area_complete() {
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    80
  // This method should do nothing.
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    81
}
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    82
#endif  // NOT_PRODUCT
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    83
489c9b5090e2 Initial load
duke
parents:
diff changeset
    84
// There may be unallocated holes in the middle chunks
22551
9bf46d16dcc6 8025856: Fix typos in the GC code
jwilhelm
parents: 22234
diff changeset
    85
// that should be filled with dead objects to ensure parsability.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    86
void MutableNUMASpace::ensure_parsability() {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    87
  for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    88
    LGRPSpace *ls = lgrp_spaces()->at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    89
    MutableSpace *s = ls->space();
2131
98f9cef66a34 6810672: Comment typos
twisti
parents: 1911
diff changeset
    90
    if (s->top() < top()) { // For all spaces preceding the one containing top()
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    91
      if (s->free_in_words() > 0) {
12229
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
    92
        intptr_t cur_top = (intptr_t)s->top();
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
    93
        size_t words_left_to_fill = pointer_delta(s->end(), s->top());;
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
    94
        while (words_left_to_fill > 0) {
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
    95
          size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size());
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
    96
          assert(words_to_fill >= CollectedHeap::min_fill_size(),
33105
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 31592
diff changeset
    97
                 "Remaining size (" SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")",
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 31592
diff changeset
    98
                 words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size());
12229
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
    99
          CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill);
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   100
          if (!os::numa_has_static_binding()) {
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   101
            size_t touched_words = words_to_fill;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   102
#ifndef ASSERT
12229
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   103
            if (!ZapUnusedHeapArea) {
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   104
              touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   105
                touched_words);
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   106
            }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   107
#endif
12229
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   108
            MemRegion invalid;
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   109
            HeapWord *crossing_start = (HeapWord*)round_to(cur_top, os::vm_page_size());
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   110
            HeapWord *crossing_end = (HeapWord*)round_to(cur_top + touched_words, os::vm_page_size());
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   111
            if (crossing_start != crossing_end) {
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   112
              // If object header crossed a small page boundary we mark the area
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   113
              // as invalid rounding it to a page_size().
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   114
              HeapWord *start = MAX2((HeapWord*)round_down(cur_top, page_size()), s->bottom());
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   115
              HeapWord *end = MIN2((HeapWord*)round_to(cur_top + touched_words, page_size()), s->end());
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   116
              invalid = MemRegion(start, end);
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   117
            }
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   118
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   119
            ls->add_invalid_region(invalid);
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   120
          }
12229
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   121
          cur_top = cur_top + (words_to_fill * HeapWordSize);
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   122
          words_left_to_fill -= words_to_fill;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   123
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   124
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   125
    } else {
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   126
      if (!os::numa_has_static_binding()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   127
#ifdef ASSERT
489c9b5090e2 Initial load
duke
parents:
diff changeset
   128
        MemRegion invalid(s->top(), s->end());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   129
        ls->add_invalid_region(invalid);
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   130
#else
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   131
        if (ZapUnusedHeapArea) {
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   132
          MemRegion invalid(s->top(), s->end());
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   133
          ls->add_invalid_region(invalid);
391
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   134
        } else {
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   135
          return;
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   136
        }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   137
#endif
391
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   138
      } else {
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   139
          return;
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   140
      }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   141
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   142
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   143
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   144
489c9b5090e2 Initial load
duke
parents:
diff changeset
   145
size_t MutableNUMASpace::used_in_words() const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   146
  size_t s = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   147
  for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   148
    s += lgrp_spaces()->at(i)->space()->used_in_words();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   149
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   150
  return s;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   151
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   152
489c9b5090e2 Initial load
duke
parents:
diff changeset
   153
size_t MutableNUMASpace::free_in_words() const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   154
  size_t s = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   155
  for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   156
    s += lgrp_spaces()->at(i)->space()->free_in_words();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   157
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   158
  return s;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   159
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   160
489c9b5090e2 Initial load
duke
parents:
diff changeset
   161
489c9b5090e2 Initial load
duke
parents:
diff changeset
   162
size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   163
  guarantee(thr != NULL, "No thread");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   164
  int lgrp_id = thr->lgrp_id();
976
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   165
  if (lgrp_id == -1) {
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   166
    // This case can occur after the topology of the system has
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   167
    // changed. Thread can change their location, the new home
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   168
    // group will be determined during the first allocation
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   169
    // attempt. For now we can safely assume that all spaces
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   170
    // have equal size because the whole space will be reinitialized.
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   171
    if (lgrp_spaces()->length() > 0) {
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   172
      return capacity_in_bytes() / lgrp_spaces()->length();
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   173
    } else {
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   174
      assert(false, "There should be at least one locality group");
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   175
      return 0;
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   176
    }
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   177
  }
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   178
  // That's the normal case, where we know the locality group of the thread.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   179
  int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   180
  if (i == -1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   181
    return 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   182
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   183
  return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   184
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   185
22552
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   186
size_t MutableNUMASpace::tlab_used(Thread *thr) const {
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   187
  // Please see the comments for tlab_capacity().
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   188
  guarantee(thr != NULL, "No thread");
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   189
  int lgrp_id = thr->lgrp_id();
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   190
  if (lgrp_id == -1) {
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   191
    if (lgrp_spaces()->length() > 0) {
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   192
      return (used_in_bytes()) / lgrp_spaces()->length();
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   193
    } else {
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   194
      assert(false, "There should be at least one locality group");
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   195
      return 0;
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   196
    }
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   197
  }
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   198
  int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   199
  if (i == -1) {
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   200
    return 0;
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   201
  }
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   202
  return lgrp_spaces()->at(i)->space()->used_in_bytes();
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   203
}
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   204
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   205
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   206
size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
976
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   207
  // Please see the comments for tlab_capacity().
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   208
  guarantee(thr != NULL, "No thread");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   209
  int lgrp_id = thr->lgrp_id();
976
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   210
  if (lgrp_id == -1) {
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   211
    if (lgrp_spaces()->length() > 0) {
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   212
      return free_in_bytes() / lgrp_spaces()->length();
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   213
    } else {
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   214
      assert(false, "There should be at least one locality group");
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   215
      return 0;
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   216
    }
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   217
  }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   218
  int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   219
  if (i == -1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   220
    return 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   221
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   222
  return lgrp_spaces()->at(i)->space()->free_in_bytes();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   223
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   224
1405
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   225
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   226
size_t MutableNUMASpace::capacity_in_words(Thread* thr) const {
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   227
  guarantee(thr != NULL, "No thread");
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   228
  int lgrp_id = thr->lgrp_id();
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   229
  if (lgrp_id == -1) {
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   230
    if (lgrp_spaces()->length() > 0) {
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   231
      return capacity_in_words() / lgrp_spaces()->length();
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   232
    } else {
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   233
      assert(false, "There should be at least one locality group");
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   234
      return 0;
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   235
    }
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   236
  }
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   237
  int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   238
  if (i == -1) {
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   239
    return 0;
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   240
  }
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   241
  return lgrp_spaces()->at(i)->space()->capacity_in_words();
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   242
}
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   243
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   244
// Check if the NUMA topology has changed. Add and remove spaces if needed.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   245
// The update can be forced by setting the force parameter equal to true.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   246
bool MutableNUMASpace::update_layout(bool force) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   247
  // Check if the topology had changed.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   248
  bool changed = os::numa_topology_changed();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   249
  if (force || changed) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   250
    // Compute lgrp intersection. Add/remove spaces.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   251
    int lgrp_limit = (int)os::numa_get_groups_num();
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 12379
diff changeset
   252
    int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit, mtGC);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   253
    int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   254
    assert(lgrp_num > 0, "There should be at least one locality group");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   255
    // Add new spaces for the new nodes
489c9b5090e2 Initial load
duke
parents:
diff changeset
   256
    for (int i = 0; i < lgrp_num; i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   257
      bool found = false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   258
      for (int j = 0; j < lgrp_spaces()->length(); j++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   259
        if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   260
          found = true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   261
          break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   262
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   263
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   264
      if (!found) {
1911
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   265
        lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment()));
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   266
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   267
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   268
489c9b5090e2 Initial load
duke
parents:
diff changeset
   269
    // Remove spaces for the removed nodes.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   270
    for (int i = 0; i < lgrp_spaces()->length();) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   271
      bool found = false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   272
      for (int j = 0; j < lgrp_num; j++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   273
        if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   274
          found = true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   275
          break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   276
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   277
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   278
      if (!found) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   279
        delete lgrp_spaces()->at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   280
        lgrp_spaces()->remove_at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   281
      } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   282
        i++;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   283
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   284
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   285
27880
afb974a04396 8060074: os::free() takes MemoryTrackingLevel but doesn't need it
coleenp
parents: 25351
diff changeset
   286
    FREE_C_HEAP_ARRAY(int, lgrp_ids);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   287
489c9b5090e2 Initial load
duke
parents:
diff changeset
   288
    if (changed) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   289
      for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   290
        thread->set_lgrp_id(-1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   291
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   292
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   293
    return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   294
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   295
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   296
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   297
489c9b5090e2 Initial load
duke
parents:
diff changeset
   298
// Bias region towards the first-touching lgrp. Set the right page sizes.
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   299
void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   300
  HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   301
  HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   302
  if (end > start) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   303
    MemRegion aligned_region(start, end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   304
    assert((intptr_t)aligned_region.start()     % page_size() == 0 &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   305
           (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   306
    assert(region().contains(aligned_region), "Sanity");
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   307
    // First we tell the OS which page size we want in the given range. The underlying
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   308
    // large page can be broken down if we require small pages.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   309
    os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   310
    // Then we uncommit the pages in the range.
11402
739e52129c84 7124829: NUMA: memory leak on Linux with large pages
iveresov
parents: 10565
diff changeset
   311
    os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   312
    // And make them local/first-touch biased.
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   313
    os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   314
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   315
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   316
489c9b5090e2 Initial load
duke
parents:
diff changeset
   317
// Free all pages in the region.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   318
void MutableNUMASpace::free_region(MemRegion mr) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   319
  HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   320
  HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   321
  if (end > start) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   322
    MemRegion aligned_region(start, end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   323
    assert((intptr_t)aligned_region.start()     % page_size() == 0 &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   324
           (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   325
    assert(region().contains(aligned_region), "Sanity");
11402
739e52129c84 7124829: NUMA: memory leak on Linux with large pages
iveresov
parents: 10565
diff changeset
   326
    os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   327
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   328
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   329
489c9b5090e2 Initial load
duke
parents:
diff changeset
   330
// Update space layout. Perform adaptation.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   331
void MutableNUMASpace::update() {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   332
  if (update_layout(false)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   333
    // If the topology has changed, make all chunks zero-sized.
976
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   334
    // And clear the alloc-rate statistics.
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   335
    // In future we may want to handle this more gracefully in order
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   336
    // to avoid the reallocation of the pages as much as possible.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   337
    for (int i = 0; i < lgrp_spaces()->length(); i++) {
976
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   338
      LGRPSpace *ls = lgrp_spaces()->at(i);
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   339
      MutableSpace *s = ls->space();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   340
      s->set_end(s->bottom());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   341
      s->set_top(s->bottom());
976
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   342
      ls->clear_alloc_rate();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   343
    }
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   344
    // A NUMA space is never mangled
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   345
    initialize(region(),
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   346
               SpaceDecorator::Clear,
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   347
               SpaceDecorator::DontMangle);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   348
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   349
    bool should_initialize = false;
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   350
    if (!os::numa_has_static_binding()) {
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   351
      for (int i = 0; i < lgrp_spaces()->length(); i++) {
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   352
        if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   353
          should_initialize = true;
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   354
          break;
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   355
        }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   356
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   357
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   358
489c9b5090e2 Initial load
duke
parents:
diff changeset
   359
    if (should_initialize ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
   360
        (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   361
      // A NUMA space is never mangled
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   362
      initialize(region(),
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   363
                 SpaceDecorator::Clear,
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   364
                 SpaceDecorator::DontMangle);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   365
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   366
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   367
489c9b5090e2 Initial load
duke
parents:
diff changeset
   368
  if (NUMAStats) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   369
    for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   370
      lgrp_spaces()->at(i)->accumulate_statistics(page_size());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   371
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   372
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   373
489c9b5090e2 Initial load
duke
parents:
diff changeset
   374
  scan_pages(NUMAPageScanRate);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   375
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   376
489c9b5090e2 Initial load
duke
parents:
diff changeset
   377
// Scan pages. Free pages that have smaller size or wrong placement.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   378
void MutableNUMASpace::scan_pages(size_t page_count)
489c9b5090e2 Initial load
duke
parents:
diff changeset
   379
{
489c9b5090e2 Initial load
duke
parents:
diff changeset
   380
  size_t pages_per_chunk = page_count / lgrp_spaces()->length();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   381
  if (pages_per_chunk > 0) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   382
    for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   383
      LGRPSpace *ls = lgrp_spaces()->at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   384
      ls->scan_pages(page_size(), pages_per_chunk);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   385
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   386
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   387
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   388
489c9b5090e2 Initial load
duke
parents:
diff changeset
   389
// Accumulate statistics about the allocation rate of each lgrp.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   390
void MutableNUMASpace::accumulate_statistics() {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   391
  if (UseAdaptiveNUMAChunkSizing) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   392
    for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   393
      lgrp_spaces()->at(i)->sample();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   394
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   395
    increment_samples_count();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   396
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   397
489c9b5090e2 Initial load
duke
parents:
diff changeset
   398
  if (NUMAStats) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   399
    for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   400
      lgrp_spaces()->at(i)->accumulate_statistics(page_size());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   401
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   402
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   403
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   404
489c9b5090e2 Initial load
duke
parents:
diff changeset
   405
// Get the current size of a chunk.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   406
// This function computes the size of the chunk based on the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   407
// difference between chunk ends. This allows it to work correctly in
489c9b5090e2 Initial load
duke
parents:
diff changeset
   408
// case the whole space is resized and during the process of adaptive
489c9b5090e2 Initial load
duke
parents:
diff changeset
   409
// chunk resizing.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   410
size_t MutableNUMASpace::current_chunk_size(int i) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   411
  HeapWord *cur_end, *prev_end;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   412
  if (i == 0) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   413
    prev_end = bottom();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   414
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   415
    prev_end = lgrp_spaces()->at(i - 1)->space()->end();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   416
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   417
  if (i == lgrp_spaces()->length() - 1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   418
    cur_end = end();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   419
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   420
    cur_end = lgrp_spaces()->at(i)->space()->end();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   421
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   422
  if (cur_end > prev_end) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   423
    return pointer_delta(cur_end, prev_end, sizeof(char));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   424
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   425
  return 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   426
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   427
489c9b5090e2 Initial load
duke
parents:
diff changeset
   428
// Return the default chunk size by equally diving the space.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   429
// page_size() aligned.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   430
size_t MutableNUMASpace::default_chunk_size() {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   431
  return base_space_size() / lgrp_spaces()->length() * page_size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   432
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   433
489c9b5090e2 Initial load
duke
parents:
diff changeset
   434
// Produce a new chunk size. page_size() aligned.
1423
1233b1e85dfd 6753547: NUMA allocator: Invalid chunk size computation during adaptive resizing
iveresov
parents: 1405
diff changeset
   435
// This function is expected to be called on sequence of i's from 0 to
1233b1e85dfd 6753547: NUMA allocator: Invalid chunk size computation during adaptive resizing
iveresov
parents: 1405
diff changeset
   436
// lgrp_spaces()->length().
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   437
size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   438
  size_t pages_available = base_space_size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   439
  for (int j = 0; j < i; j++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   440
    pages_available -= round_down(current_chunk_size(j), page_size()) / page_size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   441
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   442
  pages_available -= lgrp_spaces()->length() - i - 1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   443
  assert(pages_available > 0, "No pages left");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   444
  float alloc_rate = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   445
  for (int j = i; j < lgrp_spaces()->length(); j++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   446
    alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   447
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   448
  size_t chunk_size = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   449
  if (alloc_rate > 0) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   450
    LGRPSpace *ls = lgrp_spaces()->at(i);
1423
1233b1e85dfd 6753547: NUMA allocator: Invalid chunk size computation during adaptive resizing
iveresov
parents: 1405
diff changeset
   451
    chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   452
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   453
  chunk_size = MAX2(chunk_size, page_size());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   454
489c9b5090e2 Initial load
duke
parents:
diff changeset
   455
  if (limit > 0) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   456
    limit = round_down(limit, page_size());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   457
    if (chunk_size > current_chunk_size(i)) {
1615
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   458
      size_t upper_bound = pages_available * page_size();
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   459
      if (upper_bound > limit &&
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   460
          current_chunk_size(i) < upper_bound - limit) {
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   461
        // The resulting upper bound should not exceed the available
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   462
        // amount of memory (pages_available * page_size()).
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   463
        upper_bound = current_chunk_size(i) + limit;
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   464
      }
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   465
      chunk_size = MIN2(chunk_size, upper_bound);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   466
    } else {
1615
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   467
      size_t lower_bound = page_size();
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   468
      if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow.
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   469
        lower_bound = current_chunk_size(i) - limit;
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   470
      }
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   471
      chunk_size = MAX2(chunk_size, lower_bound);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   472
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   473
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   474
  assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   475
  return chunk_size;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   476
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   477
489c9b5090e2 Initial load
duke
parents:
diff changeset
   478
489c9b5090e2 Initial load
duke
parents:
diff changeset
   479
// Return the bottom_region and the top_region. Align them to page_size() boundary.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   480
// |------------------new_region---------------------------------|
489c9b5090e2 Initial load
duke
parents:
diff changeset
   481
// |----bottom_region--|---intersection---|------top_region------|
489c9b5090e2 Initial load
duke
parents:
diff changeset
   482
void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   483
                                    MemRegion* bottom_region, MemRegion *top_region) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   484
  // Is there bottom?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   485
  if (new_region.start() < intersection.start()) { // Yes
489c9b5090e2 Initial load
duke
parents:
diff changeset
   486
    // Try to coalesce small pages into a large one.
1911
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   487
    if (UseLargePages && page_size() >= alignment()) {
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   488
      HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), alignment());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   489
      if (new_region.contains(p)
1911
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   490
          && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   491
        if (intersection.contains(p)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   492
          intersection = MemRegion(p, intersection.end());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   493
        } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   494
          intersection = MemRegion(p, p);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   495
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   496
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   497
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   498
    *bottom_region = MemRegion(new_region.start(), intersection.start());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   499
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   500
    *bottom_region = MemRegion();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   501
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   502
489c9b5090e2 Initial load
duke
parents:
diff changeset
   503
  // Is there top?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   504
  if (intersection.end() < new_region.end()) { // Yes
489c9b5090e2 Initial load
duke
parents:
diff changeset
   505
    // Try to coalesce small pages into a large one.
1911
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   506
    if (UseLargePages && page_size() >= alignment()) {
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   507
      HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), alignment());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   508
      if (new_region.contains(p)
1911
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   509
          && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   510
        if (intersection.contains(p)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   511
          intersection = MemRegion(intersection.start(), p);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   512
        } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   513
          intersection = MemRegion(p, p);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   514
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   515
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   516
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   517
    *top_region = MemRegion(intersection.end(), new_region.end());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   518
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   519
    *top_region = MemRegion();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   520
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   521
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   522
489c9b5090e2 Initial load
duke
parents:
diff changeset
   523
// Try to merge the invalid region with the bottom or top region by decreasing
489c9b5090e2 Initial load
duke
parents:
diff changeset
   524
// the intersection area. Return the invalid_region aligned to the page_size()
489c9b5090e2 Initial load
duke
parents:
diff changeset
   525
// boundary if it's inside the intersection. Return non-empty invalid_region
489c9b5090e2 Initial load
duke
parents:
diff changeset
   526
// if it lies inside the intersection (also page-aligned).
489c9b5090e2 Initial load
duke
parents:
diff changeset
   527
// |------------------new_region---------------------------------|
489c9b5090e2 Initial load
duke
parents:
diff changeset
   528
// |----------------|-------invalid---|--------------------------|
489c9b5090e2 Initial load
duke
parents:
diff changeset
   529
// |----bottom_region--|---intersection---|------top_region------|
489c9b5090e2 Initial load
duke
parents:
diff changeset
   530
void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   531
                                     MemRegion *invalid_region) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   532
  if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   533
    *intersection = MemRegion(invalid_region->end(), intersection->end());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   534
    *invalid_region = MemRegion();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   535
  } else
489c9b5090e2 Initial load
duke
parents:
diff changeset
   536
    if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   537
      *intersection = MemRegion(intersection->start(), invalid_region->start());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   538
      *invalid_region = MemRegion();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   539
    } else
489c9b5090e2 Initial load
duke
parents:
diff changeset
   540
      if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   541
        *intersection = MemRegion(new_region.start(), new_region.start());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   542
        *invalid_region = MemRegion();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   543
      } else
489c9b5090e2 Initial load
duke
parents:
diff changeset
   544
        if (intersection->contains(invalid_region)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   545
            // That's the only case we have to make an additional bias_region() call.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   546
            HeapWord* start = invalid_region->start();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   547
            HeapWord* end = invalid_region->end();
1911
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   548
            if (UseLargePages && page_size() >= alignment()) {
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   549
              HeapWord *p = (HeapWord*)round_down((intptr_t) start, alignment());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   550
              if (new_region.contains(p)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   551
                start = p;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   552
              }
1911
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   553
              p = (HeapWord*)round_to((intptr_t) end, alignment());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   554
              if (new_region.contains(end)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   555
                end = p;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   556
              }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   557
            }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   558
            if (intersection->start() > start) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   559
              *intersection = MemRegion(start, intersection->end());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   560
            }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   561
            if (intersection->end() < end) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   562
              *intersection = MemRegion(intersection->start(), end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   563
            }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   564
            *invalid_region = MemRegion(start, end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   565
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   566
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   567
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   568
void MutableNUMASpace::initialize(MemRegion mr,
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   569
                                  bool clear_space,
1911
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   570
                                  bool mangle_space,
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   571
                                  bool setup_pages) {
22775
52bc5222f5f1 8026849: Fix typos in the GC code, part 2
jwilhelm
parents: 22552
diff changeset
   572
  assert(clear_space, "Reallocation will destroy data!");
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   573
  assert(lgrp_spaces()->length() > 0, "There should be at least one space");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   574
489c9b5090e2 Initial load
duke
parents:
diff changeset
   575
  MemRegion old_region = region(), new_region;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   576
  set_bottom(mr.start());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   577
  set_end(mr.end());
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   578
  // Must always clear the space
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   579
  clear(SpaceDecorator::DontMangle);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   580
489c9b5090e2 Initial load
duke
parents:
diff changeset
   581
  // Compute chunk sizes
489c9b5090e2 Initial load
duke
parents:
diff changeset
   582
  size_t prev_page_size = page_size();
1911
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   583
  set_page_size(UseLargePages ? alignment() : os::vm_page_size());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   584
  HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   585
  HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   586
  size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   587
489c9b5090e2 Initial load
duke
parents:
diff changeset
   588
  // Try small pages if the chunk size is too small
489c9b5090e2 Initial load
duke
parents:
diff changeset
   589
  if (base_space_size_pages / lgrp_spaces()->length() == 0
489c9b5090e2 Initial load
duke
parents:
diff changeset
   590
      && page_size() > (size_t)os::vm_page_size()) {
46312
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
   591
    // Changing the page size below can lead to freeing of memory. So we fail initialization.
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
   592
    if (_must_use_large_pages) {
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
   593
      vm_exit_during_initialization("Failed initializing NUMA with large pages. Too small heap size");
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
   594
    }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   595
    set_page_size(os::vm_page_size());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   596
    rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   597
    rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   598
    base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   599
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   600
  guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   601
  set_base_space_size(base_space_size_pages);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   602
489c9b5090e2 Initial load
duke
parents:
diff changeset
   603
  // Handle space resize
489c9b5090e2 Initial load
duke
parents:
diff changeset
   604
  MemRegion top_region, bottom_region;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   605
  if (!old_region.equals(region())) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   606
    new_region = MemRegion(rounded_bottom, rounded_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   607
    MemRegion intersection = new_region.intersection(old_region);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   608
    if (intersection.start() == NULL ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
   609
        intersection.end() == NULL   ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
   610
        prev_page_size > page_size()) { // If the page size got smaller we have to change
489c9b5090e2 Initial load
duke
parents:
diff changeset
   611
                                        // the page size preference for the whole space.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   612
      intersection = MemRegion(new_region.start(), new_region.start());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   613
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   614
    select_tails(new_region, intersection, &bottom_region, &top_region);
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   615
    bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   616
    bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   617
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   618
489c9b5090e2 Initial load
duke
parents:
diff changeset
   619
  // Check if the space layout has changed significantly?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   620
  // This happens when the space has been resized so that either head or tail
489c9b5090e2 Initial load
duke
parents:
diff changeset
   621
  // chunk became less than a page.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   622
  bool layout_valid = UseAdaptiveNUMAChunkSizing          &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   623
                      current_chunk_size(0) > page_size() &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   624
                      current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   625
489c9b5090e2 Initial load
duke
parents:
diff changeset
   626
489c9b5090e2 Initial load
duke
parents:
diff changeset
   627
  for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   628
    LGRPSpace *ls = lgrp_spaces()->at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   629
    MutableSpace *s = ls->space();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   630
    old_region = s->region();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   631
489c9b5090e2 Initial load
duke
parents:
diff changeset
   632
    size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   633
    if (i < lgrp_spaces()->length() - 1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   634
      if (!UseAdaptiveNUMAChunkSizing                                ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
   635
          (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
   636
           samples_count() < AdaptiveSizePolicyReadyThreshold) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   637
        // No adaptation. Divide the space equally.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   638
        chunk_byte_size = default_chunk_size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   639
      } else
489c9b5090e2 Initial load
duke
parents:
diff changeset
   640
        if (!layout_valid || NUMASpaceResizeRate == 0) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   641
          // Fast adaptation. If no space resize rate is set, resize
489c9b5090e2 Initial load
duke
parents:
diff changeset
   642
          // the chunks instantly.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   643
          chunk_byte_size = adaptive_chunk_size(i, 0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   644
        } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   645
          // Slow adaptation. Resize the chunks moving no more than
489c9b5090e2 Initial load
duke
parents:
diff changeset
   646
          // NUMASpaceResizeRate bytes per collection.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   647
          size_t limit = NUMASpaceResizeRate /
489c9b5090e2 Initial load
duke
parents:
diff changeset
   648
                         (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   649
          chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   650
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   651
489c9b5090e2 Initial load
duke
parents:
diff changeset
   652
      assert(chunk_byte_size >= page_size(), "Chunk size too small");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   653
      assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   654
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   655
489c9b5090e2 Initial load
duke
parents:
diff changeset
   656
    if (i == 0) { // Bottom chunk
489c9b5090e2 Initial load
duke
parents:
diff changeset
   657
      if (i != lgrp_spaces()->length() - 1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   658
        new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   659
      } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   660
        new_region = MemRegion(bottom(), end());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   661
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   662
    } else
489c9b5090e2 Initial load
duke
parents:
diff changeset
   663
      if (i < lgrp_spaces()->length() - 1) { // Middle chunks
489c9b5090e2 Initial load
duke
parents:
diff changeset
   664
        MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   665
        new_region = MemRegion(ps->end(),
489c9b5090e2 Initial load
duke
parents:
diff changeset
   666
                               ps->end() + (chunk_byte_size >> LogHeapWordSize));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   667
      } else { // Top chunk
489c9b5090e2 Initial load
duke
parents:
diff changeset
   668
        MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   669
        new_region = MemRegion(ps->end(), end());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   670
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   671
    guarantee(region().contains(new_region), "Region invariant");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   672
489c9b5090e2 Initial load
duke
parents:
diff changeset
   673
489c9b5090e2 Initial load
duke
parents:
diff changeset
   674
    // The general case:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   675
    // |---------------------|--invalid---|--------------------------|
489c9b5090e2 Initial load
duke
parents:
diff changeset
   676
    // |------------------new_region---------------------------------|
489c9b5090e2 Initial load
duke
parents:
diff changeset
   677
    // |----bottom_region--|---intersection---|------top_region------|
489c9b5090e2 Initial load
duke
parents:
diff changeset
   678
    //                     |----old_region----|
489c9b5090e2 Initial load
duke
parents:
diff changeset
   679
    // The intersection part has all pages in place we don't need to migrate them.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   680
    // Pages for the top and bottom part should be freed and then reallocated.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   681
489c9b5090e2 Initial load
duke
parents:
diff changeset
   682
    MemRegion intersection = old_region.intersection(new_region);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   683
489c9b5090e2 Initial load
duke
parents:
diff changeset
   684
    if (intersection.start() == NULL || intersection.end() == NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   685
      intersection = MemRegion(new_region.start(), new_region.start());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   686
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   687
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   688
    if (!os::numa_has_static_binding()) {
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   689
      MemRegion invalid_region = ls->invalid_region().intersection(new_region);
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   690
      // Invalid region is a range of memory that could've possibly
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   691
      // been allocated on the other node. That's relevant only on Solaris where
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   692
      // there is no static memory binding.
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   693
      if (!invalid_region.is_empty()) {
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   694
        merge_regions(new_region, &intersection, &invalid_region);
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   695
        free_region(invalid_region);
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   696
        ls->set_invalid_region(MemRegion());
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   697
      }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   698
    }
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   699
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   700
    select_tails(new_region, intersection, &bottom_region, &top_region);
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   701
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   702
    if (!os::numa_has_static_binding()) {
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   703
      // If that's a system with the first-touch policy then it's enough
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   704
      // to free the pages.
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   705
      free_region(bottom_region);
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   706
      free_region(top_region);
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   707
    } else {
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   708
      // In a system with static binding we have to change the bias whenever
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   709
      // we reshape the heap.
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   710
      bias_region(bottom_region, ls->lgrp_id());
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   711
      bias_region(top_region, ls->lgrp_id());
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   712
    }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   713
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   714
    // Clear space (set top = bottom) but never mangle.
1911
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   715
    s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   716
489c9b5090e2 Initial load
duke
parents:
diff changeset
   717
    set_adaptation_cycles(samples_count());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   718
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   719
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   720
489c9b5090e2 Initial load
duke
parents:
diff changeset
   721
// Set the top of the whole space.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   722
// Mark the the holes in chunks below the top() as invalid.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   723
void MutableNUMASpace::set_top(HeapWord* value) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   724
  bool found_top = false;
616
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   725
  for (int i = 0; i < lgrp_spaces()->length();) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   726
    LGRPSpace *ls = lgrp_spaces()->at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   727
    MutableSpace *s = ls->space();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   728
    HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   729
489c9b5090e2 Initial load
duke
parents:
diff changeset
   730
    if (s->contains(value)) {
616
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   731
      // Check if setting the chunk's top to a given value would create a hole less than
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   732
      // a minimal object; assuming that's not the last chunk in which case we don't care.
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   733
      if (i < lgrp_spaces()->length() - 1) {
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   734
        size_t remainder = pointer_delta(s->end(), value);
1668
8ec481b8f514 6578152: fill_region_with_object has usability and safety issues
jcoomes
parents: 1615
diff changeset
   735
        const size_t min_fill_size = CollectedHeap::min_fill_size();
8ec481b8f514 6578152: fill_region_with_object has usability and safety issues
jcoomes
parents: 1615
diff changeset
   736
        if (remainder < min_fill_size && remainder > 0) {
8ec481b8f514 6578152: fill_region_with_object has usability and safety issues
jcoomes
parents: 1615
diff changeset
   737
          // Add a minimum size filler object; it will cross the chunk boundary.
8ec481b8f514 6578152: fill_region_with_object has usability and safety issues
jcoomes
parents: 1615
diff changeset
   738
          CollectedHeap::fill_with_object(value, min_fill_size);
8ec481b8f514 6578152: fill_region_with_object has usability and safety issues
jcoomes
parents: 1615
diff changeset
   739
          value += min_fill_size;
616
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   740
          assert(!s->contains(value), "Should be in the next chunk");
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   741
          // Restart the loop from the same chunk, since the value has moved
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   742
          // to the next one.
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   743
          continue;
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   744
        }
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   745
      }
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   746
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   747
      if (!os::numa_has_static_binding() && top < value && top < s->end()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   748
        ls->add_invalid_region(MemRegion(top, value));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   749
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   750
      s->set_top(value);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   751
      found_top = true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   752
    } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   753
        if (found_top) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   754
            s->set_top(s->bottom());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   755
        } else {
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   756
          if (!os::numa_has_static_binding() && top < s->end()) {
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   757
            ls->add_invalid_region(MemRegion(top, s->end()));
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   758
          }
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   759
          s->set_top(s->end());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   760
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   761
    }
616
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   762
    i++;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   763
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   764
  MutableSpace::set_top(value);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   765
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   766
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   767
void MutableNUMASpace::clear(bool mangle_space) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   768
  MutableSpace::set_top(bottom());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   769
  for (int i = 0; i < lgrp_spaces()->length(); i++) {
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   770
    // Never mangle NUMA spaces because the mangling will
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   771
    // bind the memory to a possibly unwanted lgroup.
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   772
    lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   773
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   774
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   775
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   776
/*
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   777
   Linux supports static memory binding, therefore the most part of the
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   778
   logic dealing with the possible invalid page allocation is effectively
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   779
   disabled. Besides there is no notion of the home node in Linux. A
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   780
   thread is allowed to migrate freely. Although the scheduler is rather
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   781
   reluctant to move threads between the nodes. We check for the current
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   782
   node every allocation. And with a high probability a thread stays on
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   783
   the same node for some time allowing local access to recently allocated
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   784
   objects.
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   785
 */
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   786
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   787
HeapWord* MutableNUMASpace::allocate(size_t size) {
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   788
  Thread* thr = Thread::current();
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   789
  int lgrp_id = thr->lgrp_id();
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   790
  if (lgrp_id == -1 || !os::numa_has_group_homing()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   791
    lgrp_id = os::numa_get_group_id();
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   792
    thr->set_lgrp_id(lgrp_id);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   793
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   794
489c9b5090e2 Initial load
duke
parents:
diff changeset
   795
  int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   796
489c9b5090e2 Initial load
duke
parents:
diff changeset
   797
  // It is possible that a new CPU has been hotplugged and
489c9b5090e2 Initial load
duke
parents:
diff changeset
   798
  // we haven't reshaped the space accordingly.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   799
  if (i == -1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   800
    i = os::random() % lgrp_spaces()->length();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   801
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   802
1405
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   803
  LGRPSpace* ls = lgrp_spaces()->at(i);
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   804
  MutableSpace *s = ls->space();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   805
  HeapWord *p = s->allocate(size);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   806
391
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   807
  if (p != NULL) {
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   808
    size_t remainder = s->free_in_words();
5694
1e0532a6abff 6916623: Align object to 16 bytes to use Compressed Oops with java heap up to 64Gb
kvn
parents: 2154
diff changeset
   809
    if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
391
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   810
      s->set_top(s->top() - size);
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   811
      p = NULL;
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   812
    }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   813
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   814
  if (p != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   815
    if (top() < s->top()) { // Keep _top updated.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   816
      MutableSpace::set_top(s->top());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   817
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   818
  }
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   819
  // Make the page allocation happen here if there is no static binding..
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   820
  if (p != NULL && !os::numa_has_static_binding()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   821
    for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   822
      *(int*)i = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   823
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   824
  }
1405
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   825
  if (p == NULL) {
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   826
    ls->set_allocation_failed();
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   827
  }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   828
  return p;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   829
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   830
489c9b5090e2 Initial load
duke
parents:
diff changeset
   831
// This version is lock-free.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   832
HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   833
  Thread* thr = Thread::current();
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   834
  int lgrp_id = thr->lgrp_id();
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   835
  if (lgrp_id == -1 || !os::numa_has_group_homing()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   836
    lgrp_id = os::numa_get_group_id();
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   837
    thr->set_lgrp_id(lgrp_id);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   838
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   839
489c9b5090e2 Initial load
duke
parents:
diff changeset
   840
  int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   841
  // It is possible that a new CPU has been hotplugged and
489c9b5090e2 Initial load
duke
parents:
diff changeset
   842
  // we haven't reshaped the space accordingly.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   843
  if (i == -1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   844
    i = os::random() % lgrp_spaces()->length();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   845
  }
1405
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   846
  LGRPSpace *ls = lgrp_spaces()->at(i);
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   847
  MutableSpace *s = ls->space();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   848
  HeapWord *p = s->cas_allocate(size);
391
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   849
  if (p != NULL) {
616
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   850
    size_t remainder = pointer_delta(s->end(), p + size);
5694
1e0532a6abff 6916623: Align object to 16 bytes to use Compressed Oops with java heap up to 64Gb
kvn
parents: 2154
diff changeset
   851
    if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
391
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   852
      if (s->cas_deallocate(p, size)) {
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   853
        // We were the last to allocate and created a fragment less than
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   854
        // a minimal object.
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   855
        p = NULL;
616
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   856
      } else {
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   857
        guarantee(false, "Deallocation should always succeed");
391
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   858
      }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   859
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   860
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   861
  if (p != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   862
    HeapWord* cur_top, *cur_chunk_top = p + size;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   863
    while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   864
      if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   865
        break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   866
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   867
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   868
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   869
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   870
  // Make the page allocation happen here if there is no static binding.
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   871
  if (p != NULL && !os::numa_has_static_binding() ) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   872
    for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   873
      *(int*)i = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   874
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   875
  }
1405
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   876
  if (p == NULL) {
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   877
    ls->set_allocation_failed();
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   878
  }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   879
  return p;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   880
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   881
489c9b5090e2 Initial load
duke
parents:
diff changeset
   882
void MutableNUMASpace::print_short_on(outputStream* st) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   883
  MutableSpace::print_short_on(st);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   884
  st->print(" (");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   885
  for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   886
    st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   887
    lgrp_spaces()->at(i)->space()->print_short_on(st);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   888
    if (i < lgrp_spaces()->length() - 1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   889
      st->print(", ");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   890
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   891
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   892
  st->print(")");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   893
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   894
489c9b5090e2 Initial load
duke
parents:
diff changeset
   895
void MutableNUMASpace::print_on(outputStream* st) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   896
  MutableSpace::print_on(st);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   897
  for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   898
    LGRPSpace *ls = lgrp_spaces()->at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   899
    st->print("    lgrp %d", ls->lgrp_id());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   900
    ls->space()->print_on(st);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   901
    if (NUMAStats) {
391
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   902
      for (int i = 0; i < lgrp_spaces()->length(); i++) {
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   903
        lgrp_spaces()->at(i)->accumulate_statistics(page_size());
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   904
      }
24092
e274d864545a 8039743: Use correct format specifier to print size_t values and pointers in the GC code
stefank
parents: 22775
diff changeset
   905
      st->print("    local/remote/unbiased/uncommitted: " SIZE_FORMAT "K/"
e274d864545a 8039743: Use correct format specifier to print size_t values and pointers in the GC code
stefank
parents: 22775
diff changeset
   906
                SIZE_FORMAT "K/" SIZE_FORMAT "K/" SIZE_FORMAT
e274d864545a 8039743: Use correct format specifier to print size_t values and pointers in the GC code
stefank
parents: 22775
diff changeset
   907
                "K, large/small pages: " SIZE_FORMAT "/" SIZE_FORMAT "\n",
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   908
                ls->space_stats()->_local_space / K,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   909
                ls->space_stats()->_remote_space / K,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   910
                ls->space_stats()->_unbiased_space / K,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   911
                ls->space_stats()->_uncommited_space / K,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   912
                ls->space_stats()->_large_pages,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   913
                ls->space_stats()->_small_pages);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   914
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   915
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   916
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   917
12379
2cf45b79ce3a 4988100: oop_verify_old_oop appears to be dead
brutisso
parents: 12229
diff changeset
   918
void MutableNUMASpace::verify() {
22551
9bf46d16dcc6 8025856: Fix typos in the GC code
jwilhelm
parents: 22234
diff changeset
   919
  // This can be called after setting an arbitrary value to the space's top,
9bf46d16dcc6 8025856: Fix typos in the GC code
jwilhelm
parents: 22234
diff changeset
   920
  // so an object can cross the chunk boundary. We ensure the parsability
616
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   921
  // of the space and just walk the objects in linear fashion.
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   922
  ensure_parsability();
12379
2cf45b79ce3a 4988100: oop_verify_old_oop appears to be dead
brutisso
parents: 12229
diff changeset
   923
  MutableSpace::verify();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   924
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   925
489c9b5090e2 Initial load
duke
parents:
diff changeset
   926
// Scan pages and gather stats about page placement and size.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   927
void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   928
  clear_space_stats();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   929
  char *start = (char*)round_to((intptr_t) space()->bottom(), page_size);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   930
  char* end = (char*)round_down((intptr_t) space()->end(), page_size);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   931
  if (start < end) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   932
    for (char *p = start; p < end;) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   933
      os::page_info info;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   934
      if (os::get_page_info(p, &info)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   935
        if (info.size > 0) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   936
          if (info.size > (size_t)os::vm_page_size()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   937
            space_stats()->_large_pages++;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   938
          } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   939
            space_stats()->_small_pages++;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   940
          }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   941
          if (info.lgrp_id == lgrp_id()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   942
            space_stats()->_local_space += info.size;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   943
          } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   944
            space_stats()->_remote_space += info.size;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   945
          }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   946
          p += info.size;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   947
        } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   948
          p += os::vm_page_size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   949
          space_stats()->_uncommited_space += os::vm_page_size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   950
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   951
      } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   952
        return;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   953
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   954
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   955
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   956
  space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
489c9b5090e2 Initial load
duke
parents:
diff changeset
   957
                                   pointer_delta(space()->end(), end, sizeof(char));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   958
489c9b5090e2 Initial load
duke
parents:
diff changeset
   959
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   960
489c9b5090e2 Initial load
duke
parents:
diff changeset
   961
// Scan page_count pages and verify if they have the right size and right placement.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   962
// If invalid pages are found they are freed in hope that subsequent reallocation
489c9b5090e2 Initial load
duke
parents:
diff changeset
   963
// will be more successful.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   964
void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
489c9b5090e2 Initial load
duke
parents:
diff changeset
   965
{
489c9b5090e2 Initial load
duke
parents:
diff changeset
   966
  char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   967
  char* range_end = (char*)round_down((intptr_t) space()->end(), page_size);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   968
489c9b5090e2 Initial load
duke
parents:
diff changeset
   969
  if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   970
    set_last_page_scanned(range_start);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   971
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   972
489c9b5090e2 Initial load
duke
parents:
diff changeset
   973
  char *scan_start = last_page_scanned();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   974
  char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   975
489c9b5090e2 Initial load
duke
parents:
diff changeset
   976
  os::page_info page_expected, page_found;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   977
  page_expected.size = page_size;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   978
  page_expected.lgrp_id = lgrp_id();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   979
489c9b5090e2 Initial load
duke
parents:
diff changeset
   980
  char *s = scan_start;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   981
  while (s < scan_end) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   982
    char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   983
    if (e == NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   984
      break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   985
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   986
    if (e != scan_end) {
33105
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 31592
diff changeset
   987
      assert(e < scan_end, "e: " PTR_FORMAT " scan_end: " PTR_FORMAT, p2i(e), p2i(scan_end));
15955
6d0d8bea2bcc 8004697: SIGSEGV on Solaris sparc with -XX:+UseNUMA
stefank
parents: 14583
diff changeset
   988
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   989
      if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
489c9b5090e2 Initial load
duke
parents:
diff changeset
   990
          && page_expected.size != 0) {
11402
739e52129c84 7124829: NUMA: memory leak on Linux with large pages
iveresov
parents: 10565
diff changeset
   991
        os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   992
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   993
      page_expected = page_found;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   994
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   995
    s = e;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   996
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   997
489c9b5090e2 Initial load
duke
parents:
diff changeset
   998
  set_last_page_scanned(scan_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   999
}