hotspot/src/share/vm/gc/parallel/mutableNUMASpace.cpp
author stefank
Wed, 05 Jul 2017 11:33:17 +0200
changeset 46625 edefffab74e2
parent 46620 750c6edff33b
permissions -rw-r--r--
8183552: Move align functions to align.hpp Reviewed-by: stuefe, pliden
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
     1
/*
40655
9f644073d3a0 8157907: Incorrect inclusion of atomic.hpp instead of atomic.inline.hpp
dholmes
parents: 33105
diff changeset
     2
 * Copyright (c) 2006, 2016, Oracle and/or its affiliates. All rights reserved.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
     3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
489c9b5090e2 Initial load
duke
parents:
diff changeset
     4
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
     5
 * This code is free software; you can redistribute it and/or modify it
489c9b5090e2 Initial load
duke
parents:
diff changeset
     6
 * under the terms of the GNU General Public License version 2 only, as
489c9b5090e2 Initial load
duke
parents:
diff changeset
     7
 * published by the Free Software Foundation.
489c9b5090e2 Initial load
duke
parents:
diff changeset
     8
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
     9
 * This code is distributed in the hope that it will be useful, but WITHOUT
489c9b5090e2 Initial load
duke
parents:
diff changeset
    10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
489c9b5090e2 Initial load
duke
parents:
diff changeset
    11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
489c9b5090e2 Initial load
duke
parents:
diff changeset
    12
 * version 2 for more details (a copy is included in the LICENSE file that
489c9b5090e2 Initial load
duke
parents:
diff changeset
    13
 * accompanied this code).
489c9b5090e2 Initial load
duke
parents:
diff changeset
    14
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
    15
 * You should have received a copy of the GNU General Public License version
489c9b5090e2 Initial load
duke
parents:
diff changeset
    16
 * 2 along with this work; if not, write to the Free Software Foundation,
489c9b5090e2 Initial load
duke
parents:
diff changeset
    17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
489c9b5090e2 Initial load
duke
parents:
diff changeset
    18
 *
5547
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 2154
diff changeset
    19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 2154
diff changeset
    20
 * or visit www.oracle.com if you need additional information or have any
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 2154
diff changeset
    21
 * questions.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    22
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
    23
 */
489c9b5090e2 Initial load
duke
parents:
diff changeset
    24
7397
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 5702
diff changeset
    25
#include "precompiled.hpp"
30764
fec48bf5a827 8079792: GC directory structure cleanup
pliden
parents: 30154
diff changeset
    26
#include "gc/parallel/mutableNUMASpace.hpp"
fec48bf5a827 8079792: GC directory structure cleanup
pliden
parents: 30154
diff changeset
    27
#include "gc/shared/collectedHeap.hpp"
fec48bf5a827 8079792: GC directory structure cleanup
pliden
parents: 30154
diff changeset
    28
#include "gc/shared/spaceDecorator.hpp"
7397
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 5702
diff changeset
    29
#include "oops/oop.inline.hpp"
40655
9f644073d3a0 8157907: Incorrect inclusion of atomic.hpp instead of atomic.inline.hpp
dholmes
parents: 33105
diff changeset
    30
#include "runtime/atomic.hpp"
14583
d70ee55535f4 8003935: Simplify the needed includes for using Thread::current()
stefank
parents: 13195
diff changeset
    31
#include "runtime/thread.inline.hpp"
46625
edefffab74e2 8183552: Move align functions to align.hpp
stefank
parents: 46620
diff changeset
    32
#include "utilities/align.hpp"
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    33
46312
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    34
MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment), _must_use_large_pages(false) {
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 12379
diff changeset
    35
  _lgrp_spaces = new (ResourceObj::C_HEAP, mtGC) GrowableArray<LGRPSpace*>(0, true);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    36
  _page_size = os::vm_page_size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
    37
  _adaptation_cycles = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
    38
  _samples_count = 0;
46312
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    39
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    40
#ifdef LINUX
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    41
  // Changing the page size can lead to freeing of memory. When using large pages
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    42
  // and the memory has been both reserved and committed, Linux does not support
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    43
  // freeing parts of it.
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    44
    if (UseLargePages && !os::can_commit_large_page_memory()) {
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    45
      _must_use_large_pages = true;
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    46
    }
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    47
#endif // LINUX
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
    48
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    49
  update_layout(true);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    50
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
    51
489c9b5090e2 Initial load
duke
parents:
diff changeset
    52
MutableNUMASpace::~MutableNUMASpace() {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    53
  for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    54
    delete lgrp_spaces()->at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    55
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
    56
  delete lgrp_spaces();
489c9b5090e2 Initial load
duke
parents:
diff changeset
    57
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
    58
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    59
#ifndef PRODUCT
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    60
void MutableNUMASpace::mangle_unused_area() {
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    61
  // This method should do nothing.
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    62
  // It can be called on a numa space during a full compaction.
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    63
}
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    64
void MutableNUMASpace::mangle_unused_area_complete() {
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    65
  // This method should do nothing.
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    66
  // It can be called on a numa space during a full compaction.
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    67
}
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    68
void MutableNUMASpace::mangle_region(MemRegion mr) {
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    69
  // This method should do nothing because numa spaces are not mangled.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    70
}
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    71
void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    72
  assert(false, "Do not mangle MutableNUMASpace's");
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    73
}
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    74
void MutableNUMASpace::set_top_for_allocations() {
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    75
  // This method should do nothing.
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    76
}
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    77
void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    78
  // This method should do nothing.
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    79
}
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    80
void MutableNUMASpace::check_mangled_unused_area_complete() {
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    81
  // This method should do nothing.
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    82
}
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
    83
#endif  // NOT_PRODUCT
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    84
489c9b5090e2 Initial load
duke
parents:
diff changeset
    85
// There may be unallocated holes in the middle chunks
22551
9bf46d16dcc6 8025856: Fix typos in the GC code
jwilhelm
parents: 22234
diff changeset
    86
// that should be filled with dead objects to ensure parsability.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    87
void MutableNUMASpace::ensure_parsability() {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    88
  for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
    89
    LGRPSpace *ls = lgrp_spaces()->at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
    90
    MutableSpace *s = ls->space();
2131
98f9cef66a34 6810672: Comment typos
twisti
parents: 1911
diff changeset
    91
    if (s->top() < top()) { // For all spaces preceding the one containing top()
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    92
      if (s->free_in_words() > 0) {
12229
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
    93
        intptr_t cur_top = (intptr_t)s->top();
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
    94
        size_t words_left_to_fill = pointer_delta(s->end(), s->top());;
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
    95
        while (words_left_to_fill > 0) {
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
    96
          size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size());
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
    97
          assert(words_to_fill >= CollectedHeap::min_fill_size(),
33105
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 31592
diff changeset
    98
                 "Remaining size (" SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")",
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 31592
diff changeset
    99
                 words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size());
12229
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   100
          CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill);
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   101
          if (!os::numa_has_static_binding()) {
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   102
            size_t touched_words = words_to_fill;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   103
#ifndef ASSERT
12229
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   104
            if (!ZapUnusedHeapArea) {
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   105
              touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   106
                touched_words);
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   107
            }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   108
#endif
12229
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   109
            MemRegion invalid;
46620
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   110
            HeapWord *crossing_start = align_up((HeapWord*)cur_top, os::vm_page_size());
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   111
            HeapWord *crossing_end = align_down((HeapWord*)(cur_top + touched_words), os::vm_page_size());
12229
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   112
            if (crossing_start != crossing_end) {
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   113
              // If object header crossed a small page boundary we mark the area
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   114
              // as invalid rounding it to a page_size().
46620
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   115
              HeapWord *start = MAX2(align_down((HeapWord*)cur_top, page_size()), s->bottom());
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   116
              HeapWord *end = MIN2(align_up((HeapWord*)(cur_top + touched_words), page_size()), s->end());
12229
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   117
              invalid = MemRegion(start, end);
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   118
            }
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   119
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   120
            ls->add_invalid_region(invalid);
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   121
          }
12229
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   122
          cur_top = cur_top + (words_to_fill * HeapWordSize);
c34a85c8f5aa 7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents: 11402
diff changeset
   123
          words_left_to_fill -= words_to_fill;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   124
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   125
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   126
    } else {
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   127
      if (!os::numa_has_static_binding()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   128
#ifdef ASSERT
489c9b5090e2 Initial load
duke
parents:
diff changeset
   129
        MemRegion invalid(s->top(), s->end());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   130
        ls->add_invalid_region(invalid);
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   131
#else
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   132
        if (ZapUnusedHeapArea) {
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   133
          MemRegion invalid(s->top(), s->end());
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   134
          ls->add_invalid_region(invalid);
391
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   135
        } else {
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   136
          return;
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   137
        }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   138
#endif
391
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   139
      } else {
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   140
          return;
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   141
      }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   142
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   143
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   144
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   145
489c9b5090e2 Initial load
duke
parents:
diff changeset
   146
size_t MutableNUMASpace::used_in_words() const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   147
  size_t s = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   148
  for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   149
    s += lgrp_spaces()->at(i)->space()->used_in_words();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   150
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   151
  return s;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   152
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   153
489c9b5090e2 Initial load
duke
parents:
diff changeset
   154
size_t MutableNUMASpace::free_in_words() const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   155
  size_t s = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   156
  for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   157
    s += lgrp_spaces()->at(i)->space()->free_in_words();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   158
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   159
  return s;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   160
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   161
489c9b5090e2 Initial load
duke
parents:
diff changeset
   162
489c9b5090e2 Initial load
duke
parents:
diff changeset
   163
size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   164
  guarantee(thr != NULL, "No thread");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   165
  int lgrp_id = thr->lgrp_id();
976
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   166
  if (lgrp_id == -1) {
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   167
    // This case can occur after the topology of the system has
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   168
    // changed. Thread can change their location, the new home
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   169
    // group will be determined during the first allocation
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   170
    // attempt. For now we can safely assume that all spaces
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   171
    // have equal size because the whole space will be reinitialized.
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   172
    if (lgrp_spaces()->length() > 0) {
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   173
      return capacity_in_bytes() / lgrp_spaces()->length();
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   174
    } else {
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   175
      assert(false, "There should be at least one locality group");
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   176
      return 0;
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   177
    }
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   178
  }
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   179
  // That's the normal case, where we know the locality group of the thread.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   180
  int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   181
  if (i == -1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   182
    return 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   183
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   184
  return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   185
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   186
22552
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   187
size_t MutableNUMASpace::tlab_used(Thread *thr) const {
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   188
  // Please see the comments for tlab_capacity().
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   189
  guarantee(thr != NULL, "No thread");
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   190
  int lgrp_id = thr->lgrp_id();
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   191
  if (lgrp_id == -1) {
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   192
    if (lgrp_spaces()->length() > 0) {
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   193
      return (used_in_bytes()) / lgrp_spaces()->length();
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   194
    } else {
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   195
      assert(false, "There should be at least one locality group");
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   196
      return 0;
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   197
    }
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   198
  }
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   199
  int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   200
  if (i == -1) {
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   201
    return 0;
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   202
  }
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   203
  return lgrp_spaces()->at(i)->space()->used_in_bytes();
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   204
}
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   205
a29022212180 8030177: G1: Enable TLAB resizing
brutisso
parents: 22551
diff changeset
   206
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   207
size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
976
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   208
  // Please see the comments for tlab_capacity().
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   209
  guarantee(thr != NULL, "No thread");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   210
  int lgrp_id = thr->lgrp_id();
976
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   211
  if (lgrp_id == -1) {
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   212
    if (lgrp_spaces()->length() > 0) {
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   213
      return free_in_bytes() / lgrp_spaces()->length();
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   214
    } else {
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   215
      assert(false, "There should be at least one locality group");
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   216
      return 0;
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   217
    }
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   218
  }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   219
  int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   220
  if (i == -1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   221
    return 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   222
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   223
  return lgrp_spaces()->at(i)->space()->free_in_bytes();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   224
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   225
1405
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   226
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   227
size_t MutableNUMASpace::capacity_in_words(Thread* thr) const {
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   228
  guarantee(thr != NULL, "No thread");
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   229
  int lgrp_id = thr->lgrp_id();
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   230
  if (lgrp_id == -1) {
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   231
    if (lgrp_spaces()->length() > 0) {
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   232
      return capacity_in_words() / lgrp_spaces()->length();
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   233
    } else {
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   234
      assert(false, "There should be at least one locality group");
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   235
      return 0;
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   236
    }
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   237
  }
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   238
  int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   239
  if (i == -1) {
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   240
    return 0;
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   241
  }
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   242
  return lgrp_spaces()->at(i)->space()->capacity_in_words();
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   243
}
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   244
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   245
// Check if the NUMA topology has changed. Add and remove spaces if needed.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   246
// The update can be forced by setting the force parameter equal to true.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   247
bool MutableNUMASpace::update_layout(bool force) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   248
  // Check if the topology had changed.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   249
  bool changed = os::numa_topology_changed();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   250
  if (force || changed) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   251
    // Compute lgrp intersection. Add/remove spaces.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   252
    int lgrp_limit = (int)os::numa_get_groups_num();
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 12379
diff changeset
   253
    int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit, mtGC);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   254
    int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   255
    assert(lgrp_num > 0, "There should be at least one locality group");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   256
    // Add new spaces for the new nodes
489c9b5090e2 Initial load
duke
parents:
diff changeset
   257
    for (int i = 0; i < lgrp_num; i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   258
      bool found = false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   259
      for (int j = 0; j < lgrp_spaces()->length(); j++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   260
        if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   261
          found = true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   262
          break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   263
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   264
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   265
      if (!found) {
1911
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   266
        lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment()));
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   267
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   268
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   269
489c9b5090e2 Initial load
duke
parents:
diff changeset
   270
    // Remove spaces for the removed nodes.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   271
    for (int i = 0; i < lgrp_spaces()->length();) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   272
      bool found = false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   273
      for (int j = 0; j < lgrp_num; j++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   274
        if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   275
          found = true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   276
          break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   277
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   278
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   279
      if (!found) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   280
        delete lgrp_spaces()->at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   281
        lgrp_spaces()->remove_at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   282
      } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   283
        i++;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   284
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   285
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   286
27880
afb974a04396 8060074: os::free() takes MemoryTrackingLevel but doesn't need it
coleenp
parents: 25351
diff changeset
   287
    FREE_C_HEAP_ARRAY(int, lgrp_ids);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   288
489c9b5090e2 Initial load
duke
parents:
diff changeset
   289
    if (changed) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   290
      for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   291
        thread->set_lgrp_id(-1);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   292
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   293
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   294
    return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   295
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   296
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   297
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   298
489c9b5090e2 Initial load
duke
parents:
diff changeset
   299
// Bias region towards the first-touching lgrp. Set the right page sizes.
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   300
void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
46620
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   301
  HeapWord *start = align_up(mr.start(), page_size());
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   302
  HeapWord *end = align_down(mr.end(), page_size());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   303
  if (end > start) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   304
    MemRegion aligned_region(start, end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   305
    assert((intptr_t)aligned_region.start()     % page_size() == 0 &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   306
           (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   307
    assert(region().contains(aligned_region), "Sanity");
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   308
    // First we tell the OS which page size we want in the given range. The underlying
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   309
    // large page can be broken down if we require small pages.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   310
    os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   311
    // Then we uncommit the pages in the range.
11402
739e52129c84 7124829: NUMA: memory leak on Linux with large pages
iveresov
parents: 10565
diff changeset
   312
    os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   313
    // And make them local/first-touch biased.
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   314
    os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   315
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   316
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   317
489c9b5090e2 Initial load
duke
parents:
diff changeset
   318
// Free all pages in the region.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   319
void MutableNUMASpace::free_region(MemRegion mr) {
46620
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   320
  HeapWord *start = align_up(mr.start(), page_size());
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   321
  HeapWord *end = align_down(mr.end(), page_size());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   322
  if (end > start) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   323
    MemRegion aligned_region(start, end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   324
    assert((intptr_t)aligned_region.start()     % page_size() == 0 &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   325
           (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   326
    assert(region().contains(aligned_region), "Sanity");
11402
739e52129c84 7124829: NUMA: memory leak on Linux with large pages
iveresov
parents: 10565
diff changeset
   327
    os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   328
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   329
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   330
489c9b5090e2 Initial load
duke
parents:
diff changeset
   331
// Update space layout. Perform adaptation.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   332
void MutableNUMASpace::update() {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   333
  if (update_layout(false)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   334
    // If the topology has changed, make all chunks zero-sized.
976
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   335
    // And clear the alloc-rate statistics.
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   336
    // In future we may want to handle this more gracefully in order
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   337
    // to avoid the reallocation of the pages as much as possible.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   338
    for (int i = 0; i < lgrp_spaces()->length(); i++) {
976
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   339
      LGRPSpace *ls = lgrp_spaces()->at(i);
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   340
      MutableSpace *s = ls->space();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   341
      s->set_end(s->bottom());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   342
      s->set_top(s->bottom());
976
241230d48896 6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents: 971
diff changeset
   343
      ls->clear_alloc_rate();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   344
    }
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   345
    // A NUMA space is never mangled
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   346
    initialize(region(),
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   347
               SpaceDecorator::Clear,
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   348
               SpaceDecorator::DontMangle);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   349
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   350
    bool should_initialize = false;
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   351
    if (!os::numa_has_static_binding()) {
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   352
      for (int i = 0; i < lgrp_spaces()->length(); i++) {
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   353
        if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   354
          should_initialize = true;
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   355
          break;
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   356
        }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   357
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   358
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   359
489c9b5090e2 Initial load
duke
parents:
diff changeset
   360
    if (should_initialize ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
   361
        (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   362
      // A NUMA space is never mangled
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   363
      initialize(region(),
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   364
                 SpaceDecorator::Clear,
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   365
                 SpaceDecorator::DontMangle);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   366
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   367
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   368
489c9b5090e2 Initial load
duke
parents:
diff changeset
   369
  if (NUMAStats) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   370
    for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   371
      lgrp_spaces()->at(i)->accumulate_statistics(page_size());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   372
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   373
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   374
489c9b5090e2 Initial load
duke
parents:
diff changeset
   375
  scan_pages(NUMAPageScanRate);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   376
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   377
489c9b5090e2 Initial load
duke
parents:
diff changeset
   378
// Scan pages. Free pages that have smaller size or wrong placement.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   379
void MutableNUMASpace::scan_pages(size_t page_count)
489c9b5090e2 Initial load
duke
parents:
diff changeset
   380
{
489c9b5090e2 Initial load
duke
parents:
diff changeset
   381
  size_t pages_per_chunk = page_count / lgrp_spaces()->length();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   382
  if (pages_per_chunk > 0) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   383
    for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   384
      LGRPSpace *ls = lgrp_spaces()->at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   385
      ls->scan_pages(page_size(), pages_per_chunk);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   386
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   387
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   388
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   389
489c9b5090e2 Initial load
duke
parents:
diff changeset
   390
// Accumulate statistics about the allocation rate of each lgrp.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   391
void MutableNUMASpace::accumulate_statistics() {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   392
  if (UseAdaptiveNUMAChunkSizing) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   393
    for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   394
      lgrp_spaces()->at(i)->sample();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   395
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   396
    increment_samples_count();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   397
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   398
489c9b5090e2 Initial load
duke
parents:
diff changeset
   399
  if (NUMAStats) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   400
    for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   401
      lgrp_spaces()->at(i)->accumulate_statistics(page_size());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   402
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   403
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   404
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   405
489c9b5090e2 Initial load
duke
parents:
diff changeset
   406
// Get the current size of a chunk.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   407
// This function computes the size of the chunk based on the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   408
// difference between chunk ends. This allows it to work correctly in
489c9b5090e2 Initial load
duke
parents:
diff changeset
   409
// case the whole space is resized and during the process of adaptive
489c9b5090e2 Initial load
duke
parents:
diff changeset
   410
// chunk resizing.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   411
size_t MutableNUMASpace::current_chunk_size(int i) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   412
  HeapWord *cur_end, *prev_end;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   413
  if (i == 0) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   414
    prev_end = bottom();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   415
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   416
    prev_end = lgrp_spaces()->at(i - 1)->space()->end();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   417
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   418
  if (i == lgrp_spaces()->length() - 1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   419
    cur_end = end();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   420
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   421
    cur_end = lgrp_spaces()->at(i)->space()->end();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   422
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   423
  if (cur_end > prev_end) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   424
    return pointer_delta(cur_end, prev_end, sizeof(char));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   425
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   426
  return 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   427
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   428
489c9b5090e2 Initial load
duke
parents:
diff changeset
   429
// Return the default chunk size by equally diving the space.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   430
// page_size() aligned.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   431
size_t MutableNUMASpace::default_chunk_size() {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   432
  return base_space_size() / lgrp_spaces()->length() * page_size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   433
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   434
489c9b5090e2 Initial load
duke
parents:
diff changeset
   435
// Produce a new chunk size. page_size() aligned.
1423
1233b1e85dfd 6753547: NUMA allocator: Invalid chunk size computation during adaptive resizing
iveresov
parents: 1405
diff changeset
   436
// This function is expected to be called on sequence of i's from 0 to
1233b1e85dfd 6753547: NUMA allocator: Invalid chunk size computation during adaptive resizing
iveresov
parents: 1405
diff changeset
   437
// lgrp_spaces()->length().
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   438
size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   439
  size_t pages_available = base_space_size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   440
  for (int j = 0; j < i; j++) {
46620
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   441
    pages_available -= align_down(current_chunk_size(j), page_size()) / page_size();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   442
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   443
  pages_available -= lgrp_spaces()->length() - i - 1;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   444
  assert(pages_available > 0, "No pages left");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   445
  float alloc_rate = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   446
  for (int j = i; j < lgrp_spaces()->length(); j++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   447
    alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   448
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   449
  size_t chunk_size = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   450
  if (alloc_rate > 0) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   451
    LGRPSpace *ls = lgrp_spaces()->at(i);
1423
1233b1e85dfd 6753547: NUMA allocator: Invalid chunk size computation during adaptive resizing
iveresov
parents: 1405
diff changeset
   452
    chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   453
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   454
  chunk_size = MAX2(chunk_size, page_size());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   455
489c9b5090e2 Initial load
duke
parents:
diff changeset
   456
  if (limit > 0) {
46620
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   457
    limit = align_down(limit, page_size());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   458
    if (chunk_size > current_chunk_size(i)) {
1615
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   459
      size_t upper_bound = pages_available * page_size();
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   460
      if (upper_bound > limit &&
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   461
          current_chunk_size(i) < upper_bound - limit) {
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   462
        // The resulting upper bound should not exceed the available
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   463
        // amount of memory (pages_available * page_size()).
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   464
        upper_bound = current_chunk_size(i) + limit;
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   465
      }
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   466
      chunk_size = MIN2(chunk_size, upper_bound);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   467
    } else {
1615
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   468
      size_t lower_bound = page_size();
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   469
      if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow.
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   470
        lower_bound = current_chunk_size(i) - limit;
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   471
      }
b46d9f19bde2 6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents: 1423
diff changeset
   472
      chunk_size = MAX2(chunk_size, lower_bound);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   473
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   474
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   475
  assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   476
  return chunk_size;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   477
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   478
489c9b5090e2 Initial load
duke
parents:
diff changeset
   479
489c9b5090e2 Initial load
duke
parents:
diff changeset
   480
// Return the bottom_region and the top_region. Align them to page_size() boundary.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   481
// |------------------new_region---------------------------------|
489c9b5090e2 Initial load
duke
parents:
diff changeset
   482
// |----bottom_region--|---intersection---|------top_region------|
489c9b5090e2 Initial load
duke
parents:
diff changeset
   483
void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   484
                                    MemRegion* bottom_region, MemRegion *top_region) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   485
  // Is there bottom?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   486
  if (new_region.start() < intersection.start()) { // Yes
489c9b5090e2 Initial load
duke
parents:
diff changeset
   487
    // Try to coalesce small pages into a large one.
1911
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   488
    if (UseLargePages && page_size() >= alignment()) {
46620
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   489
      HeapWord* p = align_up(intersection.start(), alignment());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   490
      if (new_region.contains(p)
1911
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   491
          && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   492
        if (intersection.contains(p)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   493
          intersection = MemRegion(p, intersection.end());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   494
        } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   495
          intersection = MemRegion(p, p);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   496
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   497
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   498
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   499
    *bottom_region = MemRegion(new_region.start(), intersection.start());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   500
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   501
    *bottom_region = MemRegion();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   502
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   503
489c9b5090e2 Initial load
duke
parents:
diff changeset
   504
  // Is there top?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   505
  if (intersection.end() < new_region.end()) { // Yes
489c9b5090e2 Initial load
duke
parents:
diff changeset
   506
    // Try to coalesce small pages into a large one.
1911
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   507
    if (UseLargePages && page_size() >= alignment()) {
46620
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   508
      HeapWord* p = align_down(intersection.end(), alignment());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   509
      if (new_region.contains(p)
1911
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   510
          && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   511
        if (intersection.contains(p)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   512
          intersection = MemRegion(intersection.start(), p);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   513
        } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   514
          intersection = MemRegion(p, p);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   515
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   516
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   517
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   518
    *top_region = MemRegion(intersection.end(), new_region.end());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   519
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   520
    *top_region = MemRegion();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   521
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   522
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   523
489c9b5090e2 Initial load
duke
parents:
diff changeset
   524
// Try to merge the invalid region with the bottom or top region by decreasing
489c9b5090e2 Initial load
duke
parents:
diff changeset
   525
// the intersection area. Return the invalid_region aligned to the page_size()
489c9b5090e2 Initial load
duke
parents:
diff changeset
   526
// boundary if it's inside the intersection. Return non-empty invalid_region
489c9b5090e2 Initial load
duke
parents:
diff changeset
   527
// if it lies inside the intersection (also page-aligned).
489c9b5090e2 Initial load
duke
parents:
diff changeset
   528
// |------------------new_region---------------------------------|
489c9b5090e2 Initial load
duke
parents:
diff changeset
   529
// |----------------|-------invalid---|--------------------------|
489c9b5090e2 Initial load
duke
parents:
diff changeset
   530
// |----bottom_region--|---intersection---|------top_region------|
489c9b5090e2 Initial load
duke
parents:
diff changeset
   531
void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   532
                                     MemRegion *invalid_region) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   533
  if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   534
    *intersection = MemRegion(invalid_region->end(), intersection->end());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   535
    *invalid_region = MemRegion();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   536
  } else
489c9b5090e2 Initial load
duke
parents:
diff changeset
   537
    if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   538
      *intersection = MemRegion(intersection->start(), invalid_region->start());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   539
      *invalid_region = MemRegion();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   540
    } else
489c9b5090e2 Initial load
duke
parents:
diff changeset
   541
      if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   542
        *intersection = MemRegion(new_region.start(), new_region.start());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   543
        *invalid_region = MemRegion();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   544
      } else
489c9b5090e2 Initial load
duke
parents:
diff changeset
   545
        if (intersection->contains(invalid_region)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   546
            // That's the only case we have to make an additional bias_region() call.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   547
            HeapWord* start = invalid_region->start();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   548
            HeapWord* end = invalid_region->end();
1911
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   549
            if (UseLargePages && page_size() >= alignment()) {
46620
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   550
              HeapWord *p = align_down(start, alignment());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   551
              if (new_region.contains(p)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   552
                start = p;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   553
              }
46620
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   554
              p = align_up(end, alignment());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   555
              if (new_region.contains(end)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   556
                end = p;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   557
              }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   558
            }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   559
            if (intersection->start() > start) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   560
              *intersection = MemRegion(start, intersection->end());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   561
            }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   562
            if (intersection->end() < end) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   563
              *intersection = MemRegion(intersection->start(), end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   564
            }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   565
            *invalid_region = MemRegion(start, end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   566
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   567
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   568
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   569
void MutableNUMASpace::initialize(MemRegion mr,
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   570
                                  bool clear_space,
1911
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   571
                                  bool mangle_space,
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   572
                                  bool setup_pages) {
22775
52bc5222f5f1 8026849: Fix typos in the GC code, part 2
jwilhelm
parents: 22552
diff changeset
   573
  assert(clear_space, "Reallocation will destroy data!");
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   574
  assert(lgrp_spaces()->length() > 0, "There should be at least one space");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   575
489c9b5090e2 Initial load
duke
parents:
diff changeset
   576
  MemRegion old_region = region(), new_region;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   577
  set_bottom(mr.start());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   578
  set_end(mr.end());
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   579
  // Must always clear the space
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   580
  clear(SpaceDecorator::DontMangle);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   581
489c9b5090e2 Initial load
duke
parents:
diff changeset
   582
  // Compute chunk sizes
489c9b5090e2 Initial load
duke
parents:
diff changeset
   583
  size_t prev_page_size = page_size();
1911
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   584
  set_page_size(UseLargePages ? alignment() : os::vm_page_size());
46620
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   585
  HeapWord* rounded_bottom = align_up(bottom(), page_size());
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   586
  HeapWord* rounded_end = align_down(end(), page_size());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   587
  size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   588
489c9b5090e2 Initial load
duke
parents:
diff changeset
   589
  // Try small pages if the chunk size is too small
489c9b5090e2 Initial load
duke
parents:
diff changeset
   590
  if (base_space_size_pages / lgrp_spaces()->length() == 0
489c9b5090e2 Initial load
duke
parents:
diff changeset
   591
      && page_size() > (size_t)os::vm_page_size()) {
46312
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
   592
    // Changing the page size below can lead to freeing of memory. So we fail initialization.
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
   593
    if (_must_use_large_pages) {
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
   594
      vm_exit_during_initialization("Failed initializing NUMA with large pages. Too small heap size");
385a8b027e7d 8023905: Failing to initialize VM with small initial heap when NUMA and large pages are enabled
sangheki
parents: 40655
diff changeset
   595
    }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   596
    set_page_size(os::vm_page_size());
46620
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   597
    rounded_bottom = align_up(bottom(), page_size());
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   598
    rounded_end = align_down(end(), page_size());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   599
    base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   600
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   601
  guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   602
  set_base_space_size(base_space_size_pages);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   603
489c9b5090e2 Initial load
duke
parents:
diff changeset
   604
  // Handle space resize
489c9b5090e2 Initial load
duke
parents:
diff changeset
   605
  MemRegion top_region, bottom_region;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   606
  if (!old_region.equals(region())) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   607
    new_region = MemRegion(rounded_bottom, rounded_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   608
    MemRegion intersection = new_region.intersection(old_region);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   609
    if (intersection.start() == NULL ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
   610
        intersection.end() == NULL   ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
   611
        prev_page_size > page_size()) { // If the page size got smaller we have to change
489c9b5090e2 Initial load
duke
parents:
diff changeset
   612
                                        // the page size preference for the whole space.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   613
      intersection = MemRegion(new_region.start(), new_region.start());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   614
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   615
    select_tails(new_region, intersection, &bottom_region, &top_region);
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   616
    bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   617
    bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   618
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   619
489c9b5090e2 Initial load
duke
parents:
diff changeset
   620
  // Check if the space layout has changed significantly?
489c9b5090e2 Initial load
duke
parents:
diff changeset
   621
  // This happens when the space has been resized so that either head or tail
489c9b5090e2 Initial load
duke
parents:
diff changeset
   622
  // chunk became less than a page.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   623
  bool layout_valid = UseAdaptiveNUMAChunkSizing          &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   624
                      current_chunk_size(0) > page_size() &&
489c9b5090e2 Initial load
duke
parents:
diff changeset
   625
                      current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   626
489c9b5090e2 Initial load
duke
parents:
diff changeset
   627
489c9b5090e2 Initial load
duke
parents:
diff changeset
   628
  for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   629
    LGRPSpace *ls = lgrp_spaces()->at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   630
    MutableSpace *s = ls->space();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   631
    old_region = s->region();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   632
489c9b5090e2 Initial load
duke
parents:
diff changeset
   633
    size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   634
    if (i < lgrp_spaces()->length() - 1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   635
      if (!UseAdaptiveNUMAChunkSizing                                ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
   636
          (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
489c9b5090e2 Initial load
duke
parents:
diff changeset
   637
           samples_count() < AdaptiveSizePolicyReadyThreshold) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   638
        // No adaptation. Divide the space equally.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   639
        chunk_byte_size = default_chunk_size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   640
      } else
489c9b5090e2 Initial load
duke
parents:
diff changeset
   641
        if (!layout_valid || NUMASpaceResizeRate == 0) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   642
          // Fast adaptation. If no space resize rate is set, resize
489c9b5090e2 Initial load
duke
parents:
diff changeset
   643
          // the chunks instantly.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   644
          chunk_byte_size = adaptive_chunk_size(i, 0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   645
        } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   646
          // Slow adaptation. Resize the chunks moving no more than
489c9b5090e2 Initial load
duke
parents:
diff changeset
   647
          // NUMASpaceResizeRate bytes per collection.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   648
          size_t limit = NUMASpaceResizeRate /
489c9b5090e2 Initial load
duke
parents:
diff changeset
   649
                         (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   650
          chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   651
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   652
489c9b5090e2 Initial load
duke
parents:
diff changeset
   653
      assert(chunk_byte_size >= page_size(), "Chunk size too small");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   654
      assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   655
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   656
489c9b5090e2 Initial load
duke
parents:
diff changeset
   657
    if (i == 0) { // Bottom chunk
489c9b5090e2 Initial load
duke
parents:
diff changeset
   658
      if (i != lgrp_spaces()->length() - 1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   659
        new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   660
      } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   661
        new_region = MemRegion(bottom(), end());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   662
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   663
    } else
489c9b5090e2 Initial load
duke
parents:
diff changeset
   664
      if (i < lgrp_spaces()->length() - 1) { // Middle chunks
489c9b5090e2 Initial load
duke
parents:
diff changeset
   665
        MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   666
        new_region = MemRegion(ps->end(),
489c9b5090e2 Initial load
duke
parents:
diff changeset
   667
                               ps->end() + (chunk_byte_size >> LogHeapWordSize));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   668
      } else { // Top chunk
489c9b5090e2 Initial load
duke
parents:
diff changeset
   669
        MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   670
        new_region = MemRegion(ps->end(), end());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   671
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   672
    guarantee(region().contains(new_region), "Region invariant");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   673
489c9b5090e2 Initial load
duke
parents:
diff changeset
   674
489c9b5090e2 Initial load
duke
parents:
diff changeset
   675
    // The general case:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   676
    // |---------------------|--invalid---|--------------------------|
489c9b5090e2 Initial load
duke
parents:
diff changeset
   677
    // |------------------new_region---------------------------------|
489c9b5090e2 Initial load
duke
parents:
diff changeset
   678
    // |----bottom_region--|---intersection---|------top_region------|
489c9b5090e2 Initial load
duke
parents:
diff changeset
   679
    //                     |----old_region----|
489c9b5090e2 Initial load
duke
parents:
diff changeset
   680
    // The intersection part has all pages in place we don't need to migrate them.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   681
    // Pages for the top and bottom part should be freed and then reallocated.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   682
489c9b5090e2 Initial load
duke
parents:
diff changeset
   683
    MemRegion intersection = old_region.intersection(new_region);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   684
489c9b5090e2 Initial load
duke
parents:
diff changeset
   685
    if (intersection.start() == NULL || intersection.end() == NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   686
      intersection = MemRegion(new_region.start(), new_region.start());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   687
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   688
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   689
    if (!os::numa_has_static_binding()) {
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   690
      MemRegion invalid_region = ls->invalid_region().intersection(new_region);
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   691
      // Invalid region is a range of memory that could've possibly
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   692
      // been allocated on the other node. That's relevant only on Solaris where
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   693
      // there is no static memory binding.
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   694
      if (!invalid_region.is_empty()) {
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   695
        merge_regions(new_region, &intersection, &invalid_region);
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   696
        free_region(invalid_region);
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   697
        ls->set_invalid_region(MemRegion());
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   698
      }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   699
    }
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   700
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   701
    select_tails(new_region, intersection, &bottom_region, &top_region);
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   702
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   703
    if (!os::numa_has_static_binding()) {
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   704
      // If that's a system with the first-touch policy then it's enough
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   705
      // to free the pages.
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   706
      free_region(bottom_region);
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   707
      free_region(top_region);
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   708
    } else {
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   709
      // In a system with static binding we have to change the bias whenever
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   710
      // we reshape the heap.
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   711
      bias_region(bottom_region, ls->lgrp_id());
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   712
      bias_region(top_region, ls->lgrp_id());
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   713
    }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   714
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   715
    // Clear space (set top = bottom) but never mangle.
1911
b7cfe7eb809c 6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents: 1668
diff changeset
   716
    s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   717
489c9b5090e2 Initial load
duke
parents:
diff changeset
   718
    set_adaptation_cycles(samples_count());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   719
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   720
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   721
489c9b5090e2 Initial load
duke
parents:
diff changeset
   722
// Set the top of the whole space.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   723
// Mark the the holes in chunks below the top() as invalid.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   724
void MutableNUMASpace::set_top(HeapWord* value) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   725
  bool found_top = false;
616
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   726
  for (int i = 0; i < lgrp_spaces()->length();) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   727
    LGRPSpace *ls = lgrp_spaces()->at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   728
    MutableSpace *s = ls->space();
46620
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   729
    HeapWord *top = MAX2(align_down(s->top(), page_size()), s->bottom());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   730
489c9b5090e2 Initial load
duke
parents:
diff changeset
   731
    if (s->contains(value)) {
616
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   732
      // Check if setting the chunk's top to a given value would create a hole less than
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   733
      // a minimal object; assuming that's not the last chunk in which case we don't care.
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   734
      if (i < lgrp_spaces()->length() - 1) {
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   735
        size_t remainder = pointer_delta(s->end(), value);
1668
8ec481b8f514 6578152: fill_region_with_object has usability and safety issues
jcoomes
parents: 1615
diff changeset
   736
        const size_t min_fill_size = CollectedHeap::min_fill_size();
8ec481b8f514 6578152: fill_region_with_object has usability and safety issues
jcoomes
parents: 1615
diff changeset
   737
        if (remainder < min_fill_size && remainder > 0) {
8ec481b8f514 6578152: fill_region_with_object has usability and safety issues
jcoomes
parents: 1615
diff changeset
   738
          // Add a minimum size filler object; it will cross the chunk boundary.
8ec481b8f514 6578152: fill_region_with_object has usability and safety issues
jcoomes
parents: 1615
diff changeset
   739
          CollectedHeap::fill_with_object(value, min_fill_size);
8ec481b8f514 6578152: fill_region_with_object has usability and safety issues
jcoomes
parents: 1615
diff changeset
   740
          value += min_fill_size;
616
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   741
          assert(!s->contains(value), "Should be in the next chunk");
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   742
          // Restart the loop from the same chunk, since the value has moved
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   743
          // to the next one.
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   744
          continue;
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   745
        }
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   746
      }
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   747
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   748
      if (!os::numa_has_static_binding() && top < value && top < s->end()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   749
        ls->add_invalid_region(MemRegion(top, value));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   750
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   751
      s->set_top(value);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   752
      found_top = true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   753
    } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   754
        if (found_top) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   755
            s->set_top(s->bottom());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   756
        } else {
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   757
          if (!os::numa_has_static_binding() && top < s->end()) {
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   758
            ls->add_invalid_region(MemRegion(top, s->end()));
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   759
          }
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   760
          s->set_top(s->end());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   761
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   762
    }
616
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   763
    i++;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   764
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   765
  MutableSpace::set_top(value);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   766
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   767
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   768
void MutableNUMASpace::clear(bool mangle_space) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   769
  MutableSpace::set_top(bottom());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   770
  for (int i = 0; i < lgrp_spaces()->length(); i++) {
971
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   771
    // Never mangle NUMA spaces because the mangling will
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   772
    // bind the memory to a possibly unwanted lgroup.
f0b20be4165d 6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents: 616
diff changeset
   773
    lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   774
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   775
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   776
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   777
/*
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   778
   Linux supports static memory binding, therefore the most part of the
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   779
   logic dealing with the possible invalid page allocation is effectively
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   780
   disabled. Besides there is no notion of the home node in Linux. A
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   781
   thread is allowed to migrate freely. Although the scheduler is rather
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   782
   reluctant to move threads between the nodes. We check for the current
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   783
   node every allocation. And with a high probability a thread stays on
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   784
   the same node for some time allowing local access to recently allocated
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   785
   objects.
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   786
 */
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   787
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   788
HeapWord* MutableNUMASpace::allocate(size_t size) {
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   789
  Thread* thr = Thread::current();
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   790
  int lgrp_id = thr->lgrp_id();
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   791
  if (lgrp_id == -1 || !os::numa_has_group_homing()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   792
    lgrp_id = os::numa_get_group_id();
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   793
    thr->set_lgrp_id(lgrp_id);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   794
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   795
489c9b5090e2 Initial load
duke
parents:
diff changeset
   796
  int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   797
489c9b5090e2 Initial load
duke
parents:
diff changeset
   798
  // It is possible that a new CPU has been hotplugged and
489c9b5090e2 Initial load
duke
parents:
diff changeset
   799
  // we haven't reshaped the space accordingly.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   800
  if (i == -1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   801
    i = os::random() % lgrp_spaces()->length();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   802
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   803
1405
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   804
  LGRPSpace* ls = lgrp_spaces()->at(i);
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   805
  MutableSpace *s = ls->space();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   806
  HeapWord *p = s->allocate(size);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   807
391
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   808
  if (p != NULL) {
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   809
    size_t remainder = s->free_in_words();
5694
1e0532a6abff 6916623: Align object to 16 bytes to use Compressed Oops with java heap up to 64Gb
kvn
parents: 2154
diff changeset
   810
    if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
391
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   811
      s->set_top(s->top() - size);
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   812
      p = NULL;
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   813
    }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   814
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   815
  if (p != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   816
    if (top() < s->top()) { // Keep _top updated.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   817
      MutableSpace::set_top(s->top());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   818
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   819
  }
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   820
  // Make the page allocation happen here if there is no static binding..
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   821
  if (p != NULL && !os::numa_has_static_binding()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   822
    for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   823
      *(int*)i = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   824
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   825
  }
1405
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   826
  if (p == NULL) {
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   827
    ls->set_allocation_failed();
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   828
  }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   829
  return p;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   830
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   831
489c9b5090e2 Initial load
duke
parents:
diff changeset
   832
// This version is lock-free.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   833
HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   834
  Thread* thr = Thread::current();
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   835
  int lgrp_id = thr->lgrp_id();
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   836
  if (lgrp_id == -1 || !os::numa_has_group_homing()) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   837
    lgrp_id = os::numa_get_group_id();
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   838
    thr->set_lgrp_id(lgrp_id);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   839
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   840
489c9b5090e2 Initial load
duke
parents:
diff changeset
   841
  int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   842
  // It is possible that a new CPU has been hotplugged and
489c9b5090e2 Initial load
duke
parents:
diff changeset
   843
  // we haven't reshaped the space accordingly.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   844
  if (i == -1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   845
    i = os::random() % lgrp_spaces()->length();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   846
  }
1405
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   847
  LGRPSpace *ls = lgrp_spaces()->at(i);
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   848
  MutableSpace *s = ls->space();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   849
  HeapWord *p = s->cas_allocate(size);
391
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   850
  if (p != NULL) {
616
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   851
    size_t remainder = pointer_delta(s->end(), p + size);
5694
1e0532a6abff 6916623: Align object to 16 bytes to use Compressed Oops with java heap up to 64Gb
kvn
parents: 2154
diff changeset
   852
    if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
391
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   853
      if (s->cas_deallocate(p, size)) {
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   854
        // We were the last to allocate and created a fragment less than
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   855
        // a minimal object.
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   856
        p = NULL;
616
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   857
      } else {
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   858
        guarantee(false, "Deallocation should always succeed");
391
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   859
      }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   860
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   861
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   862
  if (p != NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   863
    HeapWord* cur_top, *cur_chunk_top = p + size;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   864
    while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   865
      if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   866
        break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   867
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   868
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   869
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   870
388
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   871
  // Make the page allocation happen here if there is no static binding.
bcc631c5bbec 6684395: Port NUMA-aware allocator to linux
iveresov
parents: 1
diff changeset
   872
  if (p != NULL && !os::numa_has_static_binding() ) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   873
    for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   874
      *(int*)i = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   875
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   876
  }
1405
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   877
  if (p == NULL) {
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   878
    ls->set_allocation_failed();
ce6e6fe90107 6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents: 977
diff changeset
   879
  }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   880
  return p;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   881
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   882
489c9b5090e2 Initial load
duke
parents:
diff changeset
   883
void MutableNUMASpace::print_short_on(outputStream* st) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   884
  MutableSpace::print_short_on(st);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   885
  st->print(" (");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   886
  for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   887
    st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   888
    lgrp_spaces()->at(i)->space()->print_short_on(st);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   889
    if (i < lgrp_spaces()->length() - 1) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   890
      st->print(", ");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   891
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   892
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   893
  st->print(")");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   894
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   895
489c9b5090e2 Initial load
duke
parents:
diff changeset
   896
void MutableNUMASpace::print_on(outputStream* st) const {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   897
  MutableSpace::print_on(st);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   898
  for (int i = 0; i < lgrp_spaces()->length(); i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   899
    LGRPSpace *ls = lgrp_spaces()->at(i);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   900
    st->print("    lgrp %d", ls->lgrp_id());
489c9b5090e2 Initial load
duke
parents:
diff changeset
   901
    ls->space()->print_on(st);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   902
    if (NUMAStats) {
391
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   903
      for (int i = 0; i < lgrp_spaces()->length(); i++) {
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   904
        lgrp_spaces()->at(i)->accumulate_statistics(page_size());
f889070a8684 6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents: 388
diff changeset
   905
      }
24092
e274d864545a 8039743: Use correct format specifier to print size_t values and pointers in the GC code
stefank
parents: 22775
diff changeset
   906
      st->print("    local/remote/unbiased/uncommitted: " SIZE_FORMAT "K/"
e274d864545a 8039743: Use correct format specifier to print size_t values and pointers in the GC code
stefank
parents: 22775
diff changeset
   907
                SIZE_FORMAT "K/" SIZE_FORMAT "K/" SIZE_FORMAT
e274d864545a 8039743: Use correct format specifier to print size_t values and pointers in the GC code
stefank
parents: 22775
diff changeset
   908
                "K, large/small pages: " SIZE_FORMAT "/" SIZE_FORMAT "\n",
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   909
                ls->space_stats()->_local_space / K,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   910
                ls->space_stats()->_remote_space / K,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   911
                ls->space_stats()->_unbiased_space / K,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   912
                ls->space_stats()->_uncommited_space / K,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   913
                ls->space_stats()->_large_pages,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   914
                ls->space_stats()->_small_pages);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   915
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   916
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   917
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   918
12379
2cf45b79ce3a 4988100: oop_verify_old_oop appears to be dead
brutisso
parents: 12229
diff changeset
   919
void MutableNUMASpace::verify() {
22551
9bf46d16dcc6 8025856: Fix typos in the GC code
jwilhelm
parents: 22234
diff changeset
   920
  // This can be called after setting an arbitrary value to the space's top,
9bf46d16dcc6 8025856: Fix typos in the GC code
jwilhelm
parents: 22234
diff changeset
   921
  // so an object can cross the chunk boundary. We ensure the parsability
616
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   922
  // of the space and just walk the objects in linear fashion.
4f2dfc0168e2 6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents: 391
diff changeset
   923
  ensure_parsability();
12379
2cf45b79ce3a 4988100: oop_verify_old_oop appears to be dead
brutisso
parents: 12229
diff changeset
   924
  MutableSpace::verify();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   925
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   926
489c9b5090e2 Initial load
duke
parents:
diff changeset
   927
// Scan pages and gather stats about page placement and size.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   928
void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   929
  clear_space_stats();
46620
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   930
  char *start = (char*)align_up(space()->bottom(), page_size);
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   931
  char* end = (char*)align_down(space()->end(), page_size);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   932
  if (start < end) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   933
    for (char *p = start; p < end;) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   934
      os::page_info info;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   935
      if (os::get_page_info(p, &info)) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   936
        if (info.size > 0) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   937
          if (info.size > (size_t)os::vm_page_size()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   938
            space_stats()->_large_pages++;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   939
          } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   940
            space_stats()->_small_pages++;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   941
          }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   942
          if (info.lgrp_id == lgrp_id()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   943
            space_stats()->_local_space += info.size;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   944
          } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   945
            space_stats()->_remote_space += info.size;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   946
          }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   947
          p += info.size;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   948
        } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   949
          p += os::vm_page_size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   950
          space_stats()->_uncommited_space += os::vm_page_size();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   951
        }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   952
      } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   953
        return;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   954
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   955
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   956
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   957
  space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
489c9b5090e2 Initial load
duke
parents:
diff changeset
   958
                                   pointer_delta(space()->end(), end, sizeof(char));
489c9b5090e2 Initial load
duke
parents:
diff changeset
   959
489c9b5090e2 Initial load
duke
parents:
diff changeset
   960
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   961
489c9b5090e2 Initial load
duke
parents:
diff changeset
   962
// Scan page_count pages and verify if they have the right size and right placement.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   963
// If invalid pages are found they are freed in hope that subsequent reallocation
489c9b5090e2 Initial load
duke
parents:
diff changeset
   964
// will be more successful.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   965
void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
489c9b5090e2 Initial load
duke
parents:
diff changeset
   966
{
46620
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   967
  char* range_start = (char*)align_up(space()->bottom(), page_size);
750c6edff33b 8178500: Replace usages of round_to and round_down with align_up and align_down
stefank
parents: 46312
diff changeset
   968
  char* range_end = (char*)align_down(space()->end(), page_size);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   969
489c9b5090e2 Initial load
duke
parents:
diff changeset
   970
  if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   971
    set_last_page_scanned(range_start);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   972
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   973
489c9b5090e2 Initial load
duke
parents:
diff changeset
   974
  char *scan_start = last_page_scanned();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   975
  char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   976
489c9b5090e2 Initial load
duke
parents:
diff changeset
   977
  os::page_info page_expected, page_found;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   978
  page_expected.size = page_size;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   979
  page_expected.lgrp_id = lgrp_id();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   980
489c9b5090e2 Initial load
duke
parents:
diff changeset
   981
  char *s = scan_start;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   982
  while (s < scan_end) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   983
    char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   984
    if (e == NULL) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   985
      break;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   986
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   987
    if (e != scan_end) {
33105
294e48b4f704 8080775: Better argument formatting for assert() and friends
david
parents: 31592
diff changeset
   988
      assert(e < scan_end, "e: " PTR_FORMAT " scan_end: " PTR_FORMAT, p2i(e), p2i(scan_end));
15955
6d0d8bea2bcc 8004697: SIGSEGV on Solaris sparc with -XX:+UseNUMA
stefank
parents: 14583
diff changeset
   989
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   990
      if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
489c9b5090e2 Initial load
duke
parents:
diff changeset
   991
          && page_expected.size != 0) {
11402
739e52129c84 7124829: NUMA: memory leak on Linux with large pages
iveresov
parents: 10565
diff changeset
   992
        os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   993
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   994
      page_expected = page_found;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   995
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   996
    s = e;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   997
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   998
489c9b5090e2 Initial load
duke
parents:
diff changeset
   999
  set_last_page_scanned(scan_end);
489c9b5090e2 Initial load
duke
parents:
diff changeset
  1000
}