hotspot/src/share/vm/utilities/taskqueue.hpp
author mikael
Tue, 09 Oct 2012 10:09:34 -0700
changeset 13963 e5b53c306fb5
parent 13195 be27e1b6a4b9
child 14292 7ee97c6fd7f1
permissions -rw-r--r--
7197424: update copyright year to match last edit in jdk8 hotspot repository Summary: Update copyright year to 2012 for relevant files Reviewed-by: dholmes, coleenp
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
     1
/*
13963
e5b53c306fb5 7197424: update copyright year to match last edit in jdk8 hotspot repository
mikael
parents: 13195
diff changeset
     2
 * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
     3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
489c9b5090e2 Initial load
duke
parents:
diff changeset
     4
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
     5
 * This code is free software; you can redistribute it and/or modify it
489c9b5090e2 Initial load
duke
parents:
diff changeset
     6
 * under the terms of the GNU General Public License version 2 only, as
489c9b5090e2 Initial load
duke
parents:
diff changeset
     7
 * published by the Free Software Foundation.
489c9b5090e2 Initial load
duke
parents:
diff changeset
     8
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
     9
 * This code is distributed in the hope that it will be useful, but WITHOUT
489c9b5090e2 Initial load
duke
parents:
diff changeset
    10
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
489c9b5090e2 Initial load
duke
parents:
diff changeset
    11
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
489c9b5090e2 Initial load
duke
parents:
diff changeset
    12
 * version 2 for more details (a copy is included in the LICENSE file that
489c9b5090e2 Initial load
duke
parents:
diff changeset
    13
 * accompanied this code).
489c9b5090e2 Initial load
duke
parents:
diff changeset
    14
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
    15
 * You should have received a copy of the GNU General Public License version
489c9b5090e2 Initial load
duke
parents:
diff changeset
    16
 * 2 along with this work; if not, write to the Free Software Foundation,
489c9b5090e2 Initial load
duke
parents:
diff changeset
    17
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
489c9b5090e2 Initial load
duke
parents:
diff changeset
    18
 *
5547
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 5076
diff changeset
    19
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 5076
diff changeset
    20
 * or visit www.oracle.com if you need additional information or have any
f4b087cbb361 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 5076
diff changeset
    21
 * questions.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
    22
 *
489c9b5090e2 Initial load
duke
parents:
diff changeset
    23
 */
489c9b5090e2 Initial load
duke
parents:
diff changeset
    24
7397
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    25
#ifndef SHARE_VM_UTILITIES_TASKQUEUE_HPP
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    26
#define SHARE_VM_UTILITIES_TASKQUEUE_HPP
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    27
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    28
#include "memory/allocation.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    29
#include "memory/allocation.inline.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    30
#include "runtime/mutex.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    31
#include "utilities/stack.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    32
#ifdef TARGET_OS_ARCH_linux_x86
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    33
# include "orderAccess_linux_x86.inline.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    34
#endif
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    35
#ifdef TARGET_OS_ARCH_linux_sparc
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    36
# include "orderAccess_linux_sparc.inline.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    37
#endif
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    38
#ifdef TARGET_OS_ARCH_linux_zero
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    39
# include "orderAccess_linux_zero.inline.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    40
#endif
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    41
#ifdef TARGET_OS_ARCH_solaris_x86
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    42
# include "orderAccess_solaris_x86.inline.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    43
#endif
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    44
#ifdef TARGET_OS_ARCH_solaris_sparc
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    45
# include "orderAccess_solaris_sparc.inline.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    46
#endif
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    47
#ifdef TARGET_OS_ARCH_windows_x86
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    48
# include "orderAccess_windows_x86.inline.hpp"
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    49
#endif
8107
78e5bd944384 7016023: Enable building ARM and PPC from src/closed repository
bobv
parents: 7397
diff changeset
    50
#ifdef TARGET_OS_ARCH_linux_arm
78e5bd944384 7016023: Enable building ARM and PPC from src/closed repository
bobv
parents: 7397
diff changeset
    51
# include "orderAccess_linux_arm.inline.hpp"
78e5bd944384 7016023: Enable building ARM and PPC from src/closed repository
bobv
parents: 7397
diff changeset
    52
#endif
78e5bd944384 7016023: Enable building ARM and PPC from src/closed repository
bobv
parents: 7397
diff changeset
    53
#ifdef TARGET_OS_ARCH_linux_ppc
78e5bd944384 7016023: Enable building ARM and PPC from src/closed repository
bobv
parents: 7397
diff changeset
    54
# include "orderAccess_linux_ppc.inline.hpp"
78e5bd944384 7016023: Enable building ARM and PPC from src/closed repository
bobv
parents: 7397
diff changeset
    55
#endif
10565
dc90c239f4ec 7089790: integrate bsd-port changes
never
parents: 8921
diff changeset
    56
#ifdef TARGET_OS_ARCH_bsd_x86
dc90c239f4ec 7089790: integrate bsd-port changes
never
parents: 8921
diff changeset
    57
# include "orderAccess_bsd_x86.inline.hpp"
dc90c239f4ec 7089790: integrate bsd-port changes
never
parents: 8921
diff changeset
    58
#endif
dc90c239f4ec 7089790: integrate bsd-port changes
never
parents: 8921
diff changeset
    59
#ifdef TARGET_OS_ARCH_bsd_zero
dc90c239f4ec 7089790: integrate bsd-port changes
never
parents: 8921
diff changeset
    60
# include "orderAccess_bsd_zero.inline.hpp"
dc90c239f4ec 7089790: integrate bsd-port changes
never
parents: 8921
diff changeset
    61
#endif
7397
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
    62
6067
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    63
// Simple TaskQueue stats that are collected by default in debug builds.
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    64
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    65
#if !defined(TASKQUEUE_STATS) && defined(ASSERT)
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    66
#define TASKQUEUE_STATS 1
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    67
#elif !defined(TASKQUEUE_STATS)
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    68
#define TASKQUEUE_STATS 0
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    69
#endif
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    70
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    71
#if TASKQUEUE_STATS
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    72
#define TASKQUEUE_STATS_ONLY(code) code
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    73
#else
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    74
#define TASKQUEUE_STATS_ONLY(code)
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    75
#endif // TASKQUEUE_STATS
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    76
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    77
#if TASKQUEUE_STATS
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    78
class TaskQueueStats {
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    79
public:
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    80
  enum StatId {
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    81
    push,             // number of taskqueue pushes
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    82
    pop,              // number of taskqueue pops
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    83
    pop_slow,         // subset of taskqueue pops that were done slow-path
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    84
    steal_attempt,    // number of taskqueue steal attempts
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    85
    steal,            // number of taskqueue steals
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    86
    overflow,         // number of overflow pushes
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    87
    overflow_max_len, // max length of overflow stack
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    88
    last_stat_id
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    89
  };
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    90
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    91
public:
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    92
  inline TaskQueueStats()       { reset(); }
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    93
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    94
  inline void record_push()     { ++_stats[push]; }
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    95
  inline void record_pop()      { ++_stats[pop]; }
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    96
  inline void record_pop_slow() { record_pop(); ++_stats[pop_slow]; }
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    97
  inline void record_steal(bool success);
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    98
  inline void record_overflow(size_t new_length);
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
    99
6251
90e562b9f1cc 6966222: G1: simplify TaskQueue overflow handling
jcoomes
parents: 6067
diff changeset
   100
  TaskQueueStats & operator +=(const TaskQueueStats & addend);
90e562b9f1cc 6966222: G1: simplify TaskQueue overflow handling
jcoomes
parents: 6067
diff changeset
   101
6067
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   102
  inline size_t get(StatId id) const { return _stats[id]; }
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   103
  inline const size_t* get() const   { return _stats; }
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   104
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   105
  inline void reset();
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   106
6251
90e562b9f1cc 6966222: G1: simplify TaskQueue overflow handling
jcoomes
parents: 6067
diff changeset
   107
  // Print the specified line of the header (does not include a line separator).
6067
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   108
  static void print_header(unsigned int line, outputStream* const stream = tty,
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   109
                           unsigned int width = 10);
6251
90e562b9f1cc 6966222: G1: simplify TaskQueue overflow handling
jcoomes
parents: 6067
diff changeset
   110
  // Print the statistics (does not include a line separator).
6067
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   111
  void print(outputStream* const stream = tty, unsigned int width = 10) const;
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   112
6251
90e562b9f1cc 6966222: G1: simplify TaskQueue overflow handling
jcoomes
parents: 6067
diff changeset
   113
  DEBUG_ONLY(void verify() const;)
90e562b9f1cc 6966222: G1: simplify TaskQueue overflow handling
jcoomes
parents: 6067
diff changeset
   114
6067
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   115
private:
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   116
  size_t                    _stats[last_stat_id];
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   117
  static const char * const _names[last_stat_id];
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   118
};
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   119
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   120
void TaskQueueStats::record_steal(bool success) {
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   121
  ++_stats[steal_attempt];
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   122
  if (success) ++_stats[steal];
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   123
}
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   124
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   125
void TaskQueueStats::record_overflow(size_t new_len) {
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   126
  ++_stats[overflow];
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   127
  if (new_len > _stats[overflow_max_len]) _stats[overflow_max_len] = new_len;
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   128
}
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   129
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   130
void TaskQueueStats::reset() {
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   131
  memset(_stats, 0, sizeof(_stats));
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   132
}
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   133
#endif // TASKQUEUE_STATS
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   134
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   135
template <unsigned int N, MEMFLAGS F>
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   136
class TaskQueueSuper: public CHeapObj<F> {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   137
protected:
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   138
  // Internal type for indexing the queue; also used for the tag.
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   139
  typedef NOT_LP64(uint16_t) LP64_ONLY(uint32_t) idx_t;
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   140
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   141
  // The first free element after the last one pushed (mod N).
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   142
  volatile uint _bottom;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   143
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   144
  enum { MOD_N_MASK = N - 1 };
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   145
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   146
  class Age {
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   147
  public:
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   148
    Age(size_t data = 0)         { _data = data; }
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   149
    Age(const Age& age)          { _data = age._data; }
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   150
    Age(idx_t top, idx_t tag)    { _fields._top = top; _fields._tag = tag; }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   151
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   152
    Age   get()        const volatile { return _data; }
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   153
    void  set(Age age) volatile       { _data = age._data; }
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   154
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   155
    idx_t top()        const volatile { return _fields._top; }
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   156
    idx_t tag()        const volatile { return _fields._tag; }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   157
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   158
    // Increment top; if it wraps, increment tag also.
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   159
    void increment() {
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   160
      _fields._top = increment_index(_fields._top);
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   161
      if (_fields._top == 0) ++_fields._tag;
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   162
    }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   163
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   164
    Age cmpxchg(const Age new_age, const Age old_age) volatile {
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   165
      return (size_t) Atomic::cmpxchg_ptr((intptr_t)new_age._data,
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   166
                                          (volatile intptr_t *)&_data,
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   167
                                          (intptr_t)old_age._data);
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   168
    }
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   169
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   170
    bool operator ==(const Age& other) const { return _data == other._data; }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   171
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   172
  private:
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   173
    struct fields {
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   174
      idx_t _top;
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   175
      idx_t _tag;
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   176
    };
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   177
    union {
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   178
      size_t _data;
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   179
      fields _fields;
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   180
    };
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   181
  };
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   182
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   183
  volatile Age _age;
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   184
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   185
  // These both operate mod N.
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   186
  static uint increment_index(uint ind) {
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   187
    return (ind + 1) & MOD_N_MASK;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   188
  }
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   189
  static uint decrement_index(uint ind) {
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   190
    return (ind - 1) & MOD_N_MASK;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   191
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   192
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   193
  // Returns a number in the range [0..N).  If the result is "N-1", it should be
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   194
  // interpreted as 0.
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   195
  uint dirty_size(uint bot, uint top) const {
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   196
    return (bot - top) & MOD_N_MASK;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   197
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   198
489c9b5090e2 Initial load
duke
parents:
diff changeset
   199
  // Returns the size corresponding to the given "bot" and "top".
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   200
  uint size(uint bot, uint top) const {
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   201
    uint sz = dirty_size(bot, top);
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   202
    // Has the queue "wrapped", so that bottom is less than top?  There's a
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   203
    // complicated special case here.  A pair of threads could perform pop_local
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   204
    // and pop_global operations concurrently, starting from a state in which
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   205
    // _bottom == _top+1.  The pop_local could succeed in decrementing _bottom,
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   206
    // and the pop_global in incrementing _top (in which case the pop_global
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   207
    // will be awarded the contested queue element.)  The resulting state must
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   208
    // be interpreted as an empty queue.  (We only need to worry about one such
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   209
    // event: only the queue owner performs pop_local's, and several concurrent
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   210
    // threads attempting to perform the pop_global will all perform the same
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   211
    // CAS, and only one can succeed.)  Any stealing thread that reads after
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   212
    // either the increment or decrement will see an empty queue, and will not
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   213
    // join the competitors.  The "sz == -1 || sz == N-1" state will not be
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   214
    // modified by concurrent queues, so the owner thread can reset the state to
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   215
    // _bottom == top so subsequent pushes will be performed normally.
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   216
    return (sz == N - 1) ? 0 : sz;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   217
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   218
489c9b5090e2 Initial load
duke
parents:
diff changeset
   219
public:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   220
  TaskQueueSuper() : _bottom(0), _age() {}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   221
5918
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   222
  // Return true if the TaskQueue contains/does not contain any tasks.
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   223
  bool peek()     const { return _bottom != _age.top(); }
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   224
  bool is_empty() const { return size() == 0; }
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   225
489c9b5090e2 Initial load
duke
parents:
diff changeset
   226
  // Return an estimate of the number of elements in the queue.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   227
  // The "careful" version admits the possibility of pop_local/pop_global
489c9b5090e2 Initial load
duke
parents:
diff changeset
   228
  // races.
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   229
  uint size() const {
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   230
    return size(_bottom, _age.top());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   231
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   232
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   233
  uint dirty_size() const {
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   234
    return dirty_size(_bottom, _age.top());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   235
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   236
1374
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   237
  void set_empty() {
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   238
    _bottom = 0;
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   239
    _age.set(0);
1374
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   240
  }
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   241
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   242
  // Maximum number of elements allowed in the queue.  This is two less
489c9b5090e2 Initial load
duke
parents:
diff changeset
   243
  // than the actual queue size, for somewhat complicated reasons.
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   244
  uint max_elems() const { return N - 2; }
5035
0e498c4df637 6928081: G1: rename parameters common with CMS
jmasa
parents: 4028
diff changeset
   245
0e498c4df637 6928081: G1: rename parameters common with CMS
jmasa
parents: 4028
diff changeset
   246
  // Total size of queue.
0e498c4df637 6928081: G1: rename parameters common with CMS
jmasa
parents: 4028
diff changeset
   247
  static const uint total_size() { return N; }
6067
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   248
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   249
  TASKQUEUE_STATS_ONLY(TaskQueueStats stats;)
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   250
};
489c9b5090e2 Initial load
duke
parents:
diff changeset
   251
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   252
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   253
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   254
template <class E, MEMFLAGS F, unsigned int N = TASKQUEUE_SIZE>
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   255
class GenericTaskQueue: public TaskQueueSuper<N, F> {
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   256
protected:
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   257
  typedef typename TaskQueueSuper<N, F>::Age Age;
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   258
  typedef typename TaskQueueSuper<N, F>::idx_t idx_t;
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   259
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   260
  using TaskQueueSuper<N, F>::_bottom;
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   261
  using TaskQueueSuper<N, F>::_age;
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   262
  using TaskQueueSuper<N, F>::increment_index;
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   263
  using TaskQueueSuper<N, F>::decrement_index;
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   264
  using TaskQueueSuper<N, F>::dirty_size;
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   265
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   266
public:
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   267
  using TaskQueueSuper<N, F>::max_elems;
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   268
  using TaskQueueSuper<N, F>::size;
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   269
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   270
#if  TASKQUEUE_STATS
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   271
  using TaskQueueSuper<N, F>::stats;
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   272
#endif
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   273
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   274
private:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   275
  // Slow paths for push, pop_local.  (pop_global has no fast path.)
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   276
  bool push_slow(E t, uint dirty_n_elems);
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   277
  bool pop_local_slow(uint localBot, Age oldAge);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   278
489c9b5090e2 Initial load
duke
parents:
diff changeset
   279
public:
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   280
  typedef E element_type;
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   281
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   282
  // Initializes the queue to empty.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   283
  GenericTaskQueue();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   284
489c9b5090e2 Initial load
duke
parents:
diff changeset
   285
  void initialize();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   286
5918
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   287
  // Push the task "t" on the queue.  Returns "false" iff the queue is full.
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   288
  inline bool push(E t);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   289
5918
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   290
  // Attempts to claim a task from the "local" end of the queue (the most
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   291
  // recently pushed).  If successful, returns true and sets t to the task;
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   292
  // otherwise, returns false (the queue is empty).
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   293
  inline bool pop_local(E& t);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   294
5918
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   295
  // Like pop_local(), but uses the "global" end of the queue (the least
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   296
  // recently pushed).
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   297
  bool pop_global(E& t);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   298
489c9b5090e2 Initial load
duke
parents:
diff changeset
   299
  // Delete any resource associated with the queue.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   300
  ~GenericTaskQueue();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   301
1374
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   302
  // apply the closure to all elements in the task queue
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   303
  void oops_do(OopClosure* f);
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   304
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   305
private:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   306
  // Element array.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   307
  volatile E* _elems;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   308
};
489c9b5090e2 Initial load
duke
parents:
diff changeset
   309
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   310
template<class E, MEMFLAGS F, unsigned int N>
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   311
GenericTaskQueue<E, F, N>::GenericTaskQueue() {
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   312
  assert(sizeof(Age) == sizeof(size_t), "Depends on this.");
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   313
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   314
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   315
template<class E, MEMFLAGS F, unsigned int N>
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   316
void GenericTaskQueue<E, F, N>::initialize() {
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   317
  _elems = NEW_C_HEAP_ARRAY(E, N, F);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   318
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   319
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   320
template<class E, MEMFLAGS F, unsigned int N>
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   321
void GenericTaskQueue<E, F, N>::oops_do(OopClosure* f) {
1374
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   322
  // tty->print_cr("START OopTaskQueue::oops_do");
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   323
  uint iters = size();
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   324
  uint index = _bottom;
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   325
  for (uint i = 0; i < iters; ++i) {
1374
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   326
    index = decrement_index(index);
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   327
    // tty->print_cr("  doing entry %d," INTPTR_T " -> " INTPTR_T,
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   328
    //            index, &_elems[index], _elems[index]);
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   329
    E* t = (E*)&_elems[index];      // cast away volatility
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   330
    oop* p = (oop*)t;
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   331
    assert((*t)->is_oop_or_null(), "Not an oop or null");
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   332
    f->do_oop(p);
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   333
  }
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   334
  // tty->print_cr("END OopTaskQueue::oops_do");
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   335
}
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   336
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   337
template<class E, MEMFLAGS F, unsigned int N>
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   338
bool GenericTaskQueue<E, F, N>::push_slow(E t, uint dirty_n_elems) {
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   339
  if (dirty_n_elems == N - 1) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   340
    // Actually means 0, so do the push.
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   341
    uint localBot = _bottom;
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   342
    // g++ complains if the volatile result of the assignment is unused.
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   343
    const_cast<E&>(_elems[localBot] = t);
4027
1c62fbaf6b68 6888847: TaskQueue needs release_store() for correctness on RMO machines
bobv
parents: 3607
diff changeset
   344
    OrderAccess::release_store(&_bottom, increment_index(localBot));
6067
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   345
    TASKQUEUE_STATS_ONLY(stats.record_push());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   346
    return true;
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   347
  }
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   348
  return false;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   349
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   350
6759
67b1a69ef5aa 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 6251
diff changeset
   351
// pop_local_slow() is done by the owning thread and is trying to
67b1a69ef5aa 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 6251
diff changeset
   352
// get the last task in the queue.  It will compete with pop_global()
67b1a69ef5aa 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 6251
diff changeset
   353
// that will be used by other threads.  The tag age is incremented
67b1a69ef5aa 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 6251
diff changeset
   354
// whenever the queue goes empty which it will do here if this thread
67b1a69ef5aa 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 6251
diff changeset
   355
// gets the last task or in pop_global() if the queue wraps (top == 0
67b1a69ef5aa 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 6251
diff changeset
   356
// and pop_global() succeeds, see pop_global()).
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   357
template<class E, MEMFLAGS F, unsigned int N>
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   358
bool GenericTaskQueue<E, F, N>::pop_local_slow(uint localBot, Age oldAge) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   359
  // This queue was observed to contain exactly one element; either this
489c9b5090e2 Initial load
duke
parents:
diff changeset
   360
  // thread will claim it, or a competing "pop_global".  In either case,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   361
  // the queue will be logically empty afterwards.  Create a new Age value
489c9b5090e2 Initial load
duke
parents:
diff changeset
   362
  // that represents the empty queue for the given value of "_bottom".  (We
489c9b5090e2 Initial load
duke
parents:
diff changeset
   363
  // must also increment "tag" because of the case where "bottom == 1",
489c9b5090e2 Initial load
duke
parents:
diff changeset
   364
  // "top == 0".  A pop_global could read the queue element in that case,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   365
  // then have the owner thread do a pop followed by another push.  Without
489c9b5090e2 Initial load
duke
parents:
diff changeset
   366
  // the incrementing of "tag", the pop_global's CAS could succeed,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   367
  // allowing it to believe it has claimed the stale element.)
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   368
  Age newAge((idx_t)localBot, oldAge.tag() + 1);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   369
  // Perhaps a competing pop_global has already incremented "top", in which
489c9b5090e2 Initial load
duke
parents:
diff changeset
   370
  // case it wins the element.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   371
  if (localBot == oldAge.top()) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   372
    // No competing pop_global has yet incremented "top"; we'll try to
489c9b5090e2 Initial load
duke
parents:
diff changeset
   373
    // install new_age, thus claiming the element.
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   374
    Age tempAge = _age.cmpxchg(newAge, oldAge);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   375
    if (tempAge == oldAge) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   376
      // We win.
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   377
      assert(dirty_size(localBot, _age.top()) != N - 1, "sanity");
6067
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   378
      TASKQUEUE_STATS_ONLY(stats.record_pop_slow());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   379
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   380
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   381
  }
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   382
  // We lose; a completing pop_global gets the element.  But the queue is empty
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   383
  // and top is greater than bottom.  Fix this representation of the empty queue
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   384
  // to become the canonical one.
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   385
  _age.set(newAge);
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   386
  assert(dirty_size(localBot, _age.top()) != N - 1, "sanity");
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   387
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   388
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   389
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   390
template<class E, MEMFLAGS F, unsigned int N>
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   391
bool GenericTaskQueue<E, F, N>::pop_global(E& t) {
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   392
  Age oldAge = _age.get();
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   393
  uint localBot = _bottom;
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   394
  uint n_elems = size(localBot, oldAge.top());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   395
  if (n_elems == 0) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   396
    return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   397
  }
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   398
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   399
  const_cast<E&>(t = _elems[oldAge.top()]);
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   400
  Age newAge(oldAge);
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   401
  newAge.increment();
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   402
  Age resAge = _age.cmpxchg(newAge, oldAge);
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   403
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   404
  // Note that using "_bottom" here might fail, since a pop_local might
489c9b5090e2 Initial load
duke
parents:
diff changeset
   405
  // have decremented it.
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   406
  assert(dirty_size(localBot, newAge.top()) != N - 1, "sanity");
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   407
  return resAge == oldAge;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   408
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   409
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   410
template<class E, MEMFLAGS F, unsigned int N>
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   411
GenericTaskQueue<E, F, N>::~GenericTaskQueue() {
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   412
  FREE_C_HEAP_ARRAY(E, _elems, F);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   413
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   414
5918
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   415
// OverflowTaskQueue is a TaskQueue that also includes an overflow stack for
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   416
// elements that do not fit in the TaskQueue.
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   417
//
6762
f8d1b560700e 6423256: GC stacks should use a better data structure
jcoomes
parents: 6759
diff changeset
   418
// This class hides two methods from super classes:
5918
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   419
//
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   420
// push() - push onto the task queue or, if that fails, onto the overflow stack
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   421
// is_empty() - return true if both the TaskQueue and overflow stack are empty
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   422
//
6762
f8d1b560700e 6423256: GC stacks should use a better data structure
jcoomes
parents: 6759
diff changeset
   423
// Note that size() is not hidden--it returns the number of elements in the
5918
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   424
// TaskQueue, and does not include the size of the overflow stack.  This
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   425
// simplifies replacement of GenericTaskQueues with OverflowTaskQueues.
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   426
template<class E, MEMFLAGS F, unsigned int N = TASKQUEUE_SIZE>
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   427
class OverflowTaskQueue: public GenericTaskQueue<E, F, N>
5918
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   428
{
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   429
public:
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   430
  typedef Stack<E, F>               overflow_t;
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   431
  typedef GenericTaskQueue<E, F, N> taskqueue_t;
5918
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   432
6067
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   433
  TASKQUEUE_STATS_ONLY(using taskqueue_t::stats;)
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   434
5918
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   435
  // Push task t onto the queue or onto the overflow stack.  Return true.
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   436
  inline bool push(E t);
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   437
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   438
  // Attempt to pop from the overflow stack; return true if anything was popped.
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   439
  inline bool pop_overflow(E& t);
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   440
6762
f8d1b560700e 6423256: GC stacks should use a better data structure
jcoomes
parents: 6759
diff changeset
   441
  inline overflow_t* overflow_stack() { return &_overflow_stack; }
f8d1b560700e 6423256: GC stacks should use a better data structure
jcoomes
parents: 6759
diff changeset
   442
5918
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   443
  inline bool taskqueue_empty() const { return taskqueue_t::is_empty(); }
6762
f8d1b560700e 6423256: GC stacks should use a better data structure
jcoomes
parents: 6759
diff changeset
   444
  inline bool overflow_empty()  const { return _overflow_stack.is_empty(); }
5918
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   445
  inline bool is_empty()        const {
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   446
    return taskqueue_empty() && overflow_empty();
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   447
  }
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   448
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   449
private:
6762
f8d1b560700e 6423256: GC stacks should use a better data structure
jcoomes
parents: 6759
diff changeset
   450
  overflow_t _overflow_stack;
5918
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   451
};
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   452
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   453
template <class E, MEMFLAGS F, unsigned int N>
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   454
bool OverflowTaskQueue<E, F, N>::push(E t)
5918
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   455
{
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   456
  if (!taskqueue_t::push(t)) {
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   457
    overflow_stack()->push(t);
6762
f8d1b560700e 6423256: GC stacks should use a better data structure
jcoomes
parents: 6759
diff changeset
   458
    TASKQUEUE_STATS_ONLY(stats.record_overflow(overflow_stack()->size()));
5918
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   459
  }
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   460
  return true;
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   461
}
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   462
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   463
template <class E, MEMFLAGS F, unsigned int N>
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   464
bool OverflowTaskQueue<E, F, N>::pop_overflow(E& t)
5918
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   465
{
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   466
  if (overflow_empty()) return false;
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   467
  t = overflow_stack()->pop();
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   468
  return true;
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   469
}
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   470
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   471
class TaskQueueSetSuper {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   472
protected:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   473
  static int randomParkAndMiller(int* seed0);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   474
public:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   475
  // Returns "true" if some TaskQueue in the set contains a task.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   476
  virtual bool peek() = 0;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   477
};
489c9b5090e2 Initial load
duke
parents:
diff changeset
   478
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   479
template <MEMFLAGS F> class TaskQueueSetSuperImpl: public CHeapObj<F>, public TaskQueueSetSuper {
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   480
};
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   481
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   482
template<class T, MEMFLAGS F>
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   483
class GenericTaskQueueSet: public TaskQueueSetSuperImpl<F> {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   484
private:
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   485
  uint _n;
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   486
  T** _queues;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   487
489c9b5090e2 Initial load
duke
parents:
diff changeset
   488
public:
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   489
  typedef typename T::element_type E;
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   490
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   491
  GenericTaskQueueSet(int n) : _n(n) {
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   492
    typedef T* GenericTaskQueuePtr;
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   493
    _queues = NEW_C_HEAP_ARRAY(GenericTaskQueuePtr, n, F);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   494
    for (int i = 0; i < n; i++) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   495
      _queues[i] = NULL;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   496
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   497
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   498
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   499
  bool steal_1_random(uint queue_num, int* seed, E& t);
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   500
  bool steal_best_of_2(uint queue_num, int* seed, E& t);
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   501
  bool steal_best_of_all(uint queue_num, int* seed, E& t);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   502
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   503
  void register_queue(uint i, T* q);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   504
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   505
  T* queue(uint n);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   506
5918
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   507
  // The thread with queue number "queue_num" (and whose random number seed is
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   508
  // at "seed") is trying to steal a task from some other queue.  (It may try
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   509
  // several queues, according to some configuration parameter.)  If some steal
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   510
  // succeeds, returns "true" and sets "t" to the stolen task, otherwise returns
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   511
  // false.
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   512
  bool steal(uint queue_num, int* seed, E& t);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   513
489c9b5090e2 Initial load
duke
parents:
diff changeset
   514
  bool peek();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   515
};
489c9b5090e2 Initial load
duke
parents:
diff changeset
   516
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   517
template<class T, MEMFLAGS F> void
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   518
GenericTaskQueueSet<T, F>::register_queue(uint i, T* q) {
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   519
  assert(i < _n, "index out of range.");
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   520
  _queues[i] = q;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   521
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   522
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   523
template<class T, MEMFLAGS F> T*
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   524
GenericTaskQueueSet<T, F>::queue(uint i) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   525
  return _queues[i];
489c9b5090e2 Initial load
duke
parents:
diff changeset
   526
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   527
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   528
template<class T, MEMFLAGS F> bool
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   529
GenericTaskQueueSet<T, F>::steal(uint queue_num, int* seed, E& t) {
6067
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   530
  for (uint i = 0; i < 2 * _n; i++) {
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   531
    if (steal_best_of_2(queue_num, seed, t)) {
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   532
      TASKQUEUE_STATS_ONLY(queue(queue_num)->stats.record_steal(true));
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   533
      return true;
6067
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   534
    }
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   535
  }
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   536
  TASKQUEUE_STATS_ONLY(queue(queue_num)->stats.record_steal(false));
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   537
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   538
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   539
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   540
template<class T, MEMFLAGS F> bool
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   541
GenericTaskQueueSet<T, F>::steal_best_of_all(uint queue_num, int* seed, E& t) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   542
  if (_n > 2) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   543
    int best_k;
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   544
    uint best_sz = 0;
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   545
    for (uint k = 0; k < _n; k++) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   546
      if (k == queue_num) continue;
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   547
      uint sz = _queues[k]->size();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   548
      if (sz > best_sz) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   549
        best_sz = sz;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   550
        best_k = k;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   551
      }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   552
    }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   553
    return best_sz > 0 && _queues[best_k]->pop_global(t);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   554
  } else if (_n == 2) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   555
    // Just try the other one.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   556
    int k = (queue_num + 1) % 2;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   557
    return _queues[k]->pop_global(t);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   558
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   559
    assert(_n == 1, "can't be zero.");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   560
    return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   561
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   562
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   563
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   564
template<class T, MEMFLAGS F> bool
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   565
GenericTaskQueueSet<T, F>::steal_1_random(uint queue_num, int* seed, E& t) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   566
  if (_n > 2) {
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   567
    uint k = queue_num;
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   568
    while (k == queue_num) k = TaskQueueSetSuper::randomParkAndMiller(seed) % _n;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   569
    return _queues[2]->pop_global(t);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   570
  } else if (_n == 2) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   571
    // Just try the other one.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   572
    int k = (queue_num + 1) % 2;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   573
    return _queues[k]->pop_global(t);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   574
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   575
    assert(_n == 1, "can't be zero.");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   576
    return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   577
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   578
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   579
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   580
template<class T, MEMFLAGS F> bool
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   581
GenericTaskQueueSet<T, F>::steal_best_of_2(uint queue_num, int* seed, E& t) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   582
  if (_n > 2) {
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   583
    uint k1 = queue_num;
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   584
    while (k1 == queue_num) k1 = TaskQueueSetSuper::randomParkAndMiller(seed) % _n;
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   585
    uint k2 = queue_num;
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   586
    while (k2 == queue_num || k2 == k1) k2 = TaskQueueSetSuper::randomParkAndMiller(seed) % _n;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   587
    // Sample both and try the larger.
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   588
    uint sz1 = _queues[k1]->size();
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   589
    uint sz2 = _queues[k2]->size();
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   590
    if (sz2 > sz1) return _queues[k2]->pop_global(t);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   591
    else return _queues[k1]->pop_global(t);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   592
  } else if (_n == 2) {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   593
    // Just try the other one.
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   594
    uint k = (queue_num + 1) % 2;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   595
    return _queues[k]->pop_global(t);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   596
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   597
    assert(_n == 1, "can't be zero.");
489c9b5090e2 Initial load
duke
parents:
diff changeset
   598
    return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   599
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   600
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   601
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   602
template<class T, MEMFLAGS F>
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   603
bool GenericTaskQueueSet<T, F>::peek() {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   604
  // Try all the queues.
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   605
  for (uint j = 0; j < _n; j++) {
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   606
    if (_queues[j]->peek())
489c9b5090e2 Initial load
duke
parents:
diff changeset
   607
      return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   608
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   609
  return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   610
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   611
1374
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   612
// When to terminate from the termination protocol.
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   613
class TerminatorTerminator: public CHeapObj<mtInternal> {
1374
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   614
public:
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   615
  virtual bool should_exit_termination() = 0;
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   616
};
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   617
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   618
// A class to aid in the termination of a set of parallel tasks using
489c9b5090e2 Initial load
duke
parents:
diff changeset
   619
// TaskQueueSet's for work stealing.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   620
2010
c13462bbad17 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 2005
diff changeset
   621
#undef TRACESPINNING
c13462bbad17 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 2005
diff changeset
   622
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   623
class ParallelTaskTerminator: public StackObj {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   624
private:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   625
  int _n_threads;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   626
  TaskQueueSetSuper* _queue_set;
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   627
  int _offered_termination;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   628
2010
c13462bbad17 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 2005
diff changeset
   629
#ifdef TRACESPINNING
c13462bbad17 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 2005
diff changeset
   630
  static uint _total_yields;
c13462bbad17 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 2005
diff changeset
   631
  static uint _total_spins;
c13462bbad17 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 2005
diff changeset
   632
  static uint _total_peeks;
c13462bbad17 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 2005
diff changeset
   633
#endif
c13462bbad17 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 2005
diff changeset
   634
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   635
  bool peek_in_queue_set();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   636
protected:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   637
  virtual void yield();
489c9b5090e2 Initial load
duke
parents:
diff changeset
   638
  void sleep(uint millis);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   639
489c9b5090e2 Initial load
duke
parents:
diff changeset
   640
public:
489c9b5090e2 Initial load
duke
parents:
diff changeset
   641
489c9b5090e2 Initial load
duke
parents:
diff changeset
   642
  // "n_threads" is the number of threads to be terminated.  "queue_set" is a
489c9b5090e2 Initial load
duke
parents:
diff changeset
   643
  // queue sets of work queues of other threads.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   644
  ParallelTaskTerminator(int n_threads, TaskQueueSetSuper* queue_set);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   645
489c9b5090e2 Initial load
duke
parents:
diff changeset
   646
  // The current thread has no work, and is ready to terminate if everyone
489c9b5090e2 Initial load
duke
parents:
diff changeset
   647
  // else is.  If returns "true", all threads are terminated.  If returns
489c9b5090e2 Initial load
duke
parents:
diff changeset
   648
  // "false", available work has been observed in one of the task queues,
489c9b5090e2 Initial load
duke
parents:
diff changeset
   649
  // so the global task is not complete.
1374
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   650
  bool offer_termination() {
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   651
    return offer_termination(NULL);
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   652
  }
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   653
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   654
  // As above, but it also terminates if the should_exit_termination()
1374
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   655
  // method of the terminator parameter returns true. If terminator is
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   656
  // NULL, then it is ignored.
4c24294029a9 6711316: Open source the Garbage-First garbage collector
ysr
parents: 360
diff changeset
   657
  bool offer_termination(TerminatorTerminator* terminator);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   658
489c9b5090e2 Initial load
duke
parents:
diff changeset
   659
  // Reset the terminator, so that it may be reused again.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   660
  // The caller is responsible for ensuring that this is done
489c9b5090e2 Initial load
duke
parents:
diff changeset
   661
  // in an MT-safe manner, once the previous round of use of
489c9b5090e2 Initial load
duke
parents:
diff changeset
   662
  // the terminator is finished.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   663
  void reset_for_reuse();
6759
67b1a69ef5aa 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 6251
diff changeset
   664
  // Same as above but the number of parallel threads is set to the
67b1a69ef5aa 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 6251
diff changeset
   665
  // given number.
67b1a69ef5aa 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 6251
diff changeset
   666
  void reset_for_reuse(int n_threads);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   667
2010
c13462bbad17 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 2005
diff changeset
   668
#ifdef TRACESPINNING
c13462bbad17 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 2005
diff changeset
   669
  static uint total_yields() { return _total_yields; }
c13462bbad17 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 2005
diff changeset
   670
  static uint total_spins() { return _total_spins; }
c13462bbad17 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 2005
diff changeset
   671
  static uint total_peeks() { return _total_peeks; }
c13462bbad17 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 2005
diff changeset
   672
  static void print_termination_counts();
c13462bbad17 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 2005
diff changeset
   673
#endif
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   674
};
489c9b5090e2 Initial load
duke
parents:
diff changeset
   675
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   676
template<class E, MEMFLAGS F, unsigned int N> inline bool
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   677
GenericTaskQueue<E, F, N>::push(E t) {
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   678
  uint localBot = _bottom;
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   679
  assert((localBot >= 0) && (localBot < N), "_bottom out of range.");
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   680
  idx_t top = _age.top();
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   681
  uint dirty_n_elems = dirty_size(localBot, top);
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   682
  assert(dirty_n_elems < N, "n_elems out of range.");
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   683
  if (dirty_n_elems < max_elems()) {
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   684
    // g++ complains if the volatile result of the assignment is unused.
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   685
    const_cast<E&>(_elems[localBot] = t);
4027
1c62fbaf6b68 6888847: TaskQueue needs release_store() for correctness on RMO machines
bobv
parents: 3607
diff changeset
   686
    OrderAccess::release_store(&_bottom, increment_index(localBot));
6067
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   687
    TASKQUEUE_STATS_ONLY(stats.record_push());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   688
    return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   689
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   690
    return push_slow(t, dirty_n_elems);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   691
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   692
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   693
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   694
template<class E, MEMFLAGS F, unsigned int N> inline bool
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   695
GenericTaskQueue<E, F, N>::pop_local(E& t) {
2005
42075507972b 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 1407
diff changeset
   696
  uint localBot = _bottom;
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   697
  // This value cannot be N-1.  That can only occur as a result of
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   698
  // the assignment to bottom in this method.  If it does, this method
5918
73b96456819a 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 5547
diff changeset
   699
  // resets the size to 0 before the next call (which is sequential,
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   700
  // since this is pop_local.)
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   701
  uint dirty_n_elems = dirty_size(localBot, _age.top());
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   702
  assert(dirty_n_elems != N - 1, "Shouldn't be possible...");
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   703
  if (dirty_n_elems == 0) return false;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   704
  localBot = decrement_index(localBot);
489c9b5090e2 Initial load
duke
parents:
diff changeset
   705
  _bottom = localBot;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   706
  // This is necessary to prevent any read below from being reordered
489c9b5090e2 Initial load
duke
parents:
diff changeset
   707
  // before the store just above.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   708
  OrderAccess::fence();
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   709
  const_cast<E&>(t = _elems[localBot]);
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   710
  // This is a second read of "age"; the "size()" above is the first.
489c9b5090e2 Initial load
duke
parents:
diff changeset
   711
  // If there's still at least one element in the queue, based on the
489c9b5090e2 Initial load
duke
parents:
diff changeset
   712
  // "_bottom" and "age" we've read, then there can be no interference with
489c9b5090e2 Initial load
duke
parents:
diff changeset
   713
  // a "pop_global" operation, and we're done.
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   714
  idx_t tp = _age.top();    // XXX
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   715
  if (size(localBot, tp) > 0) {
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   716
    assert(dirty_size(localBot, tp) != N - 1, "sanity");
6067
8bfddf73fc04 6962947: shared TaskQueue statistics
jcoomes
parents: 5918
diff changeset
   717
    TASKQUEUE_STATS_ONLY(stats.record_pop());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   718
    return true;
489c9b5090e2 Initial load
duke
parents:
diff changeset
   719
  } else {
489c9b5090e2 Initial load
duke
parents:
diff changeset
   720
    // Otherwise, the queue contained exactly one element; we take the slow
489c9b5090e2 Initial load
duke
parents:
diff changeset
   721
    // path.
3607
fa62293fa65c 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 3262
diff changeset
   722
    return pop_local_slow(localBot, _age.get());
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   723
  }
489c9b5090e2 Initial load
duke
parents:
diff changeset
   724
}
489c9b5090e2 Initial load
duke
parents:
diff changeset
   725
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   726
typedef GenericTaskQueue<oop, mtGC>             OopTaskQueue;
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   727
typedef GenericTaskQueueSet<OopTaskQueue, mtGC> OopTaskQueueSet;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   728
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   729
#ifdef _MSC_VER
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   730
#pragma warning(push)
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   731
// warning C4522: multiple assignment operators specified
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   732
#pragma warning(disable:4522)
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   733
#endif
360
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 1
diff changeset
   734
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 1
diff changeset
   735
// This is a container class for either an oop* or a narrowOop*.
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 1
diff changeset
   736
// Both are pushed onto a task queue and the consumer will test is_narrow()
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 1
diff changeset
   737
// to determine which should be processed.
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 1
diff changeset
   738
class StarTask {
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 1
diff changeset
   739
  void*  _holder;        // either union oop* or narrowOop*
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   740
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   741
  enum { COMPRESSED_OOP_MASK = 1 };
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   742
360
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 1
diff changeset
   743
 public:
3262
30d1c247fc25 6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents: 2105
diff changeset
   744
  StarTask(narrowOop* p) {
30d1c247fc25 6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents: 2105
diff changeset
   745
    assert(((uintptr_t)p & COMPRESSED_OOP_MASK) == 0, "Information loss!");
30d1c247fc25 6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents: 2105
diff changeset
   746
    _holder = (void *)((uintptr_t)p | COMPRESSED_OOP_MASK);
30d1c247fc25 6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents: 2105
diff changeset
   747
  }
30d1c247fc25 6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents: 2105
diff changeset
   748
  StarTask(oop* p)       {
30d1c247fc25 6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents: 2105
diff changeset
   749
    assert(((uintptr_t)p & COMPRESSED_OOP_MASK) == 0, "Information loss!");
30d1c247fc25 6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents: 2105
diff changeset
   750
    _holder = (void*)p;
30d1c247fc25 6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents: 2105
diff changeset
   751
  }
360
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 1
diff changeset
   752
  StarTask()             { _holder = NULL; }
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 1
diff changeset
   753
  operator oop*()        { return (oop*)_holder; }
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 1
diff changeset
   754
  operator narrowOop*()  {
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 1
diff changeset
   755
    return (narrowOop*)((uintptr_t)_holder & ~COMPRESSED_OOP_MASK);
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 1
diff changeset
   756
  }
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 1
diff changeset
   757
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   758
  StarTask& operator=(const StarTask& t) {
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   759
    _holder = t._holder;
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   760
    return *this;
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   761
  }
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   762
  volatile StarTask& operator=(const volatile StarTask& t) volatile {
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   763
    _holder = t._holder;
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   764
    return *this;
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   765
  }
360
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 1
diff changeset
   766
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 1
diff changeset
   767
  bool is_narrow() const {
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 1
diff changeset
   768
    return (((uintptr_t)_holder & COMPRESSED_OOP_MASK) != 0);
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 1
diff changeset
   769
  }
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 1
diff changeset
   770
};
21d113ecbf6a 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 1
diff changeset
   771
5076
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   772
class ObjArrayTask
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   773
{
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   774
public:
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   775
  ObjArrayTask(oop o = NULL, int idx = 0): _obj(o), _index(idx) { }
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   776
  ObjArrayTask(oop o, size_t idx): _obj(o), _index(int(idx)) {
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   777
    assert(idx <= size_t(max_jint), "too big");
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   778
  }
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   779
  ObjArrayTask(const ObjArrayTask& t): _obj(t._obj), _index(t._index) { }
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   780
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   781
  ObjArrayTask& operator =(const ObjArrayTask& t) {
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   782
    _obj = t._obj;
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   783
    _index = t._index;
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   784
    return *this;
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   785
  }
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   786
  volatile ObjArrayTask&
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   787
  operator =(const volatile ObjArrayTask& t) volatile {
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   788
    _obj = t._obj;
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   789
    _index = t._index;
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   790
    return *this;
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   791
  }
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   792
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   793
  inline oop obj()   const { return _obj; }
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   794
  inline int index() const { return _index; }
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   795
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   796
  DEBUG_ONLY(bool is_valid() const); // Tasks to be pushed/popped must be valid.
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   797
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   798
private:
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   799
  oop _obj;
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   800
  int _index;
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   801
};
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   802
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   803
#ifdef _MSC_VER
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   804
#pragma warning(pop)
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   805
#endif
8b74a4b60b31 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 5035
diff changeset
   806
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   807
typedef OverflowTaskQueue<StarTask, mtClass>           OopStarTaskQueue;
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   808
typedef GenericTaskQueueSet<OopStarTaskQueue, mtClass> OopStarTaskQueueSet;
1
489c9b5090e2 Initial load
duke
parents:
diff changeset
   809
13195
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   810
typedef OverflowTaskQueue<size_t, mtInternal>             RegionTaskQueue;
be27e1b6a4b9 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 10565
diff changeset
   811
typedef GenericTaskQueueSet<RegionTaskQueue, mtClass>     RegionTaskQueueSet;
6759
67b1a69ef5aa 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 6251
diff changeset
   812
7397
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
   813
5b173b4ca846 6989984: Use standard include model for Hospot
stefank
parents: 6762
diff changeset
   814
#endif // SHARE_VM_UTILITIES_TASKQUEUE_HPP