8192943: Optimize atomic accumulators using VarHandle getAndSet
Reviewed-by: martin, psandoz, chegar
--- a/src/java.base/share/classes/java/util/concurrent/atomic/DoubleAccumulator.java Fri Dec 08 15:22:58 2017 -0800
+++ b/src/java.base/share/classes/java/util/concurrent/atomic/DoubleAccumulator.java Fri Dec 08 15:26:56 2017 -0800
@@ -105,20 +105,20 @@
* @param x the value
*/
public void accumulate(double x) {
- Cell[] as; long b, v, r; int m; Cell a;
- if ((as = cells) != null
+ Cell[] cs; long b, v, r; int m; Cell c;
+ if ((cs = cells) != null
|| ((r = doubleToRawLongBits
(function.applyAsDouble(longBitsToDouble(b = base), x))) != b
&& !casBase(b, r))) {
boolean uncontended = true;
- if (as == null
- || (m = as.length - 1) < 0
- || (a = as[getProbe() & m]) == null
+ if (cs == null
+ || (m = cs.length - 1) < 0
+ || (c = cs[getProbe() & m]) == null
|| !(uncontended =
((r = doubleToRawLongBits
(function.applyAsDouble
- (longBitsToDouble(v = a.value), x))) == v)
- || a.cas(v, r)))
+ (longBitsToDouble(v = c.value), x))) == v)
+ || c.cas(v, r)))
doubleAccumulate(x, function, uncontended);
}
}
@@ -133,13 +133,13 @@
* @return the current value
*/
public double get() {
- Cell[] as = cells;
+ Cell[] cs = cells;
double result = longBitsToDouble(base);
- if (as != null) {
- for (Cell a : as)
- if (a != null)
+ if (cs != null) {
+ for (Cell c : cs)
+ if (c != null)
result = function.applyAsDouble
- (result, longBitsToDouble(a.value));
+ (result, longBitsToDouble(c.value));
}
return result;
}
@@ -153,12 +153,12 @@
* updating.
*/
public void reset() {
- Cell[] as = cells;
+ Cell[] cs = cells;
base = identity;
- if (as != null) {
- for (Cell a : as)
- if (a != null)
- a.reset(identity);
+ if (cs != null) {
+ for (Cell c : cs)
+ if (c != null)
+ c.reset(identity);
}
}
@@ -173,14 +173,12 @@
* @return the value before reset
*/
public double getThenReset() {
- Cell[] as = cells;
- double result = longBitsToDouble(base);
- base = identity;
- if (as != null) {
- for (Cell a : as) {
- if (a != null) {
- double v = longBitsToDouble(a.value);
- a.reset(identity);
+ Cell[] cs = cells;
+ double result = longBitsToDouble(getAndSetBase(identity));
+ if (cs != null) {
+ for (Cell c : cs) {
+ if (c != null) {
+ double v = longBitsToDouble(c.getAndSet(identity));
result = function.applyAsDouble(result, v);
}
}
--- a/src/java.base/share/classes/java/util/concurrent/atomic/DoubleAdder.java Fri Dec 08 15:22:58 2017 -0800
+++ b/src/java.base/share/classes/java/util/concurrent/atomic/DoubleAdder.java Fri Dec 08 15:26:56 2017 -0800
@@ -87,15 +87,15 @@
* @param x the value to add
*/
public void add(double x) {
- Cell[] as; long b, v; int m; Cell a;
- if ((as = cells) != null ||
+ Cell[] cs; long b, v; int m; Cell c;
+ if ((cs = cells) != null ||
!casBase(b = base,
Double.doubleToRawLongBits
(Double.longBitsToDouble(b) + x))) {
boolean uncontended = true;
- if (as == null || (m = as.length - 1) < 0 ||
- (a = as[getProbe() & m]) == null ||
- !(uncontended = a.cas(v = a.value,
+ if (cs == null || (m = cs.length - 1) < 0 ||
+ (c = cs[getProbe() & m]) == null ||
+ !(uncontended = c.cas(v = c.value,
Double.doubleToRawLongBits
(Double.longBitsToDouble(v) + x))))
doubleAccumulate(x, null, uncontended);
@@ -115,12 +115,12 @@
* @return the sum
*/
public double sum() {
- Cell[] as = cells;
+ Cell[] cs = cells;
double sum = Double.longBitsToDouble(base);
- if (as != null) {
- for (Cell a : as)
- if (a != null)
- sum += Double.longBitsToDouble(a.value);
+ if (cs != null) {
+ for (Cell c : cs)
+ if (c != null)
+ sum += Double.longBitsToDouble(c.value);
}
return sum;
}
@@ -133,12 +133,12 @@
* known that no threads are concurrently updating.
*/
public void reset() {
- Cell[] as = cells;
+ Cell[] cs = cells;
base = 0L; // relies on fact that double 0 must have same rep as long
- if (as != null) {
- for (Cell a : as)
- if (a != null)
- a.reset();
+ if (cs != null) {
+ for (Cell c : cs)
+ if (c != null)
+ c.reset();
}
}
@@ -153,16 +153,12 @@
* @return the sum
*/
public double sumThenReset() {
- Cell[] as = cells;
- double sum = Double.longBitsToDouble(base);
- base = 0L;
- if (as != null) {
- for (Cell a : as) {
- if (a != null) {
- long v = a.value;
- a.reset();
- sum += Double.longBitsToDouble(v);
- }
+ Cell[] cs = cells;
+ double sum = Double.longBitsToDouble(getAndSetBase(0L));
+ if (cs != null) {
+ for (Cell c : cs) {
+ if (c != null)
+ sum += Double.longBitsToDouble(c.getAndSet(0L));
}
}
return sum;
--- a/src/java.base/share/classes/java/util/concurrent/atomic/LongAccumulator.java Fri Dec 08 15:22:58 2017 -0800
+++ b/src/java.base/share/classes/java/util/concurrent/atomic/LongAccumulator.java Fri Dec 08 15:26:56 2017 -0800
@@ -103,17 +103,17 @@
* @param x the value
*/
public void accumulate(long x) {
- Cell[] as; long b, v, r; int m; Cell a;
- if ((as = cells) != null
+ Cell[] cs; long b, v, r; int m; Cell c;
+ if ((cs = cells) != null
|| ((r = function.applyAsLong(b = base, x)) != b
&& !casBase(b, r))) {
boolean uncontended = true;
- if (as == null
- || (m = as.length - 1) < 0
- || (a = as[getProbe() & m]) == null
+ if (cs == null
+ || (m = cs.length - 1) < 0
+ || (c = cs[getProbe() & m]) == null
|| !(uncontended =
- (r = function.applyAsLong(v = a.value, x)) == v
- || a.cas(v, r)))
+ (r = function.applyAsLong(v = c.value, x)) == v
+ || c.cas(v, r)))
longAccumulate(x, function, uncontended);
}
}
@@ -128,12 +128,12 @@
* @return the current value
*/
public long get() {
- Cell[] as = cells;
+ Cell[] cs = cells;
long result = base;
- if (as != null) {
- for (Cell a : as)
- if (a != null)
- result = function.applyAsLong(result, a.value);
+ if (cs != null) {
+ for (Cell c : cs)
+ if (c != null)
+ result = function.applyAsLong(result, c.value);
}
return result;
}
@@ -147,12 +147,12 @@
* updating.
*/
public void reset() {
- Cell[] as = cells;
+ Cell[] cs = cells;
base = identity;
- if (as != null) {
- for (Cell a : as)
- if (a != null)
- a.reset(identity);
+ if (cs != null) {
+ for (Cell c : cs)
+ if (c != null)
+ c.reset(identity);
}
}
@@ -167,14 +167,12 @@
* @return the value before reset
*/
public long getThenReset() {
- Cell[] as = cells;
- long result = base;
- base = identity;
- if (as != null) {
- for (Cell a : as) {
- if (a != null) {
- long v = a.value;
- a.reset(identity);
+ Cell[] cs = cells;
+ long result = getAndSetBase(identity);
+ if (cs != null) {
+ for (Cell c : cs) {
+ if (c != null) {
+ long v = c.getAndSet(identity);
result = function.applyAsLong(result, v);
}
}
--- a/src/java.base/share/classes/java/util/concurrent/atomic/LongAdder.java Fri Dec 08 15:22:58 2017 -0800
+++ b/src/java.base/share/classes/java/util/concurrent/atomic/LongAdder.java Fri Dec 08 15:26:56 2017 -0800
@@ -83,12 +83,12 @@
* @param x the value to add
*/
public void add(long x) {
- Cell[] as; long b, v; int m; Cell a;
- if ((as = cells) != null || !casBase(b = base, b + x)) {
+ Cell[] cs; long b, v; int m; Cell c;
+ if ((cs = cells) != null || !casBase(b = base, b + x)) {
boolean uncontended = true;
- if (as == null || (m = as.length - 1) < 0 ||
- (a = as[getProbe() & m]) == null ||
- !(uncontended = a.cas(v = a.value, v + x)))
+ if (cs == null || (m = cs.length - 1) < 0 ||
+ (c = cs[getProbe() & m]) == null ||
+ !(uncontended = c.cas(v = c.value, v + x)))
longAccumulate(x, null, uncontended);
}
}
@@ -117,12 +117,12 @@
* @return the sum
*/
public long sum() {
- Cell[] as = cells;
+ Cell[] cs = cells;
long sum = base;
- if (as != null) {
- for (Cell a : as)
- if (a != null)
- sum += a.value;
+ if (cs != null) {
+ for (Cell c : cs)
+ if (c != null)
+ sum += c.value;
}
return sum;
}
@@ -135,12 +135,12 @@
* known that no threads are concurrently updating.
*/
public void reset() {
- Cell[] as = cells;
+ Cell[] cs = cells;
base = 0L;
- if (as != null) {
- for (Cell a : as)
- if (a != null)
- a.reset();
+ if (cs != null) {
+ for (Cell c : cs)
+ if (c != null)
+ c.reset();
}
}
@@ -155,15 +155,12 @@
* @return the sum
*/
public long sumThenReset() {
- Cell[] as = cells;
- long sum = base;
- base = 0L;
- if (as != null) {
- for (Cell a : as) {
- if (a != null) {
- sum += a.value;
- a.reset();
- }
+ Cell[] cs = cells;
+ long sum = getAndSetBase(0L);
+ if (cs != null) {
+ for (Cell c : cs) {
+ if (c != null)
+ sum += c.getAndSet(0L);
}
}
return sum;
--- a/src/java.base/share/classes/java/util/concurrent/atomic/Striped64.java Fri Dec 08 15:22:58 2017 -0800
+++ b/src/java.base/share/classes/java/util/concurrent/atomic/Striped64.java Fri Dec 08 15:26:56 2017 -0800
@@ -133,6 +133,9 @@
final void reset(long identity) {
VALUE.setVolatile(this, identity);
}
+ final long getAndSet(long val) {
+ return (long)VALUE.getAndSet(this, val);
+ }
// VarHandle mechanics
private static final VarHandle VALUE;
@@ -178,6 +181,10 @@
return BASE.compareAndSet(this, cmp, val);
}
+ final long getAndSetBase(long val) {
+ return (long)BASE.getAndSet(this, val);
+ }
+
/**
* CASes the cellsBusy field from 0 to 1 to acquire lock.
*/
@@ -228,9 +235,9 @@
}
boolean collide = false; // True if last slot nonempty
done: for (;;) {
- Cell[] as; Cell a; int n; long v;
- if ((as = cells) != null && (n = as.length) > 0) {
- if ((a = as[(n - 1) & h]) == null) {
+ Cell[] cs; Cell c; int n; long v;
+ if ((cs = cells) != null && (n = cs.length) > 0) {
+ if ((c = cs[(n - 1) & h]) == null) {
if (cellsBusy == 0) { // Try to attach new Cell
Cell r = new Cell(x); // Optimistically create
if (cellsBusy == 0 && casCellsBusy()) {
@@ -252,17 +259,17 @@
}
else if (!wasUncontended) // CAS already known to fail
wasUncontended = true; // Continue after rehash
- else if (a.cas(v = a.value,
+ else if (c.cas(v = c.value,
(fn == null) ? v + x : fn.applyAsLong(v, x)))
break;
- else if (n >= NCPU || cells != as)
+ else if (n >= NCPU || cells != cs)
collide = false; // At max size or stale
else if (!collide)
collide = true;
else if (cellsBusy == 0 && casCellsBusy()) {
try {
- if (cells == as) // Expand table unless stale
- cells = Arrays.copyOf(as, n << 1);
+ if (cells == cs) // Expand table unless stale
+ cells = Arrays.copyOf(cs, n << 1);
} finally {
cellsBusy = 0;
}
@@ -271,9 +278,9 @@
}
h = advanceProbe(h);
}
- else if (cellsBusy == 0 && cells == as && casCellsBusy()) {
+ else if (cellsBusy == 0 && cells == cs && casCellsBusy()) {
try { // Initialize table
- if (cells == as) {
+ if (cells == cs) {
Cell[] rs = new Cell[2];
rs[h & 1] = new Cell(x);
cells = rs;
@@ -312,9 +319,9 @@
}
boolean collide = false; // True if last slot nonempty
done: for (;;) {
- Cell[] as; Cell a; int n; long v;
- if ((as = cells) != null && (n = as.length) > 0) {
- if ((a = as[(n - 1) & h]) == null) {
+ Cell[] cs; Cell c; int n; long v;
+ if ((cs = cells) != null && (n = cs.length) > 0) {
+ if ((c = cs[(n - 1) & h]) == null) {
if (cellsBusy == 0) { // Try to attach new Cell
Cell r = new Cell(Double.doubleToRawLongBits(x));
if (cellsBusy == 0 && casCellsBusy()) {
@@ -336,16 +343,16 @@
}
else if (!wasUncontended) // CAS already known to fail
wasUncontended = true; // Continue after rehash
- else if (a.cas(v = a.value, apply(fn, v, x)))
+ else if (c.cas(v = c.value, apply(fn, v, x)))
break;
- else if (n >= NCPU || cells != as)
+ else if (n >= NCPU || cells != cs)
collide = false; // At max size or stale
else if (!collide)
collide = true;
else if (cellsBusy == 0 && casCellsBusy()) {
try {
- if (cells == as) // Expand table unless stale
- cells = Arrays.copyOf(as, n << 1);
+ if (cells == cs) // Expand table unless stale
+ cells = Arrays.copyOf(cs, n << 1);
} finally {
cellsBusy = 0;
}
@@ -354,9 +361,9 @@
}
h = advanceProbe(h);
}
- else if (cellsBusy == 0 && cells == as && casCellsBusy()) {
+ else if (cellsBusy == 0 && cells == cs && casCellsBusy()) {
try { // Initialize table
- if (cells == as) {
+ if (cells == cs) {
Cell[] rs = new Cell[2];
rs[h & 1] = new Cell(Double.doubleToRawLongBits(x));
cells = rs;