8014219: Make the run-octane harness more deterministic by not measuring elapsed time every iteration. Also got rid of most of the run logic in base.js and call benchmarks directly for the same purpose
Reviewed-by: jlaskey, attila
--- a/nashorn/make/build-benchmark.xml Fri May 24 23:27:52 2013 +0530
+++ b/nashorn/make/build-benchmark.xml Mon May 27 13:11:13 2013 +0200
@@ -32,250 +32,263 @@
<pathconvert pathsep=" " property="octane-tests" refid="octane-set"/>
</target>
+
<!-- box2d -->
<target name="octane-box2d" depends="jar">
<antcall target="run-octane">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/box2d.js"/>
+ <param name="octane-tests" value="box2d"/>
</antcall>
</target>
<target name="octane-box2d-v8" depends="jar">
<antcall target="run-octane-v8">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/box2d.js"/>
+ <param name="octane-tests" value="box2d"/>
</antcall>
</target>
<target name="octane-box2d-rhino" depends="jar">
<antcall target="run-octane-rhino">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/box2d.js"/>
+ <param name="octane-tests" value="box2d"/>
</antcall>
</target>
+
<!-- code-load -->
<target name="octane-code-load" depends="jar">
<antcall target="run-octane">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/code-load.js"/>
+ <param name="octane-tests" value="code-load"/>
</antcall>
</target>
<target name="octane-code-load-v8" depends="jar">
<antcall target="run-octane-v8">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/code-load.js"/>
+ <param name="octane-tests" value="code-load"/>
</antcall>
</target>
<target name="octane-code-load-rhino" depends="jar">
<antcall target="run-octane-rhino">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/code-load.js"/>
+ <param name="octane-tests" value="code-load"/>
</antcall>
</target>
+
<!-- crypto -->
<target name="octane-crypto" depends="jar">
<antcall target="run-octane">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/crypto.js"/>
+ <param name="octane-tests" value="crypto"/>
</antcall>
</target>
<target name="octane-crypto-v8" depends="jar">
<antcall target="run-octane-v8">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/crypto.js"/>
+ <param name="octane-tests" value="crypto"/>
</antcall>
</target>
<target name="octane-crypto-rhino" depends="jar">
<antcall target="run-octane-rhino">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/crypto.js"/>
+ <param name="octane-tests" value="crypto"/>
</antcall>
</target>
+
<!-- deltablue -->
<target name="octane-deltablue" depends="jar">
<antcall target="run-octane">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/deltablue.js"/>
+ <param name="octane-tests" value="deltablue"/>
</antcall>
</target>
<target name="octane-deltablue-v8" depends="jar">
<antcall target="run-octane-v8">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/deltablue.js"/>
+ <param name="octane-tests" value="deltablue"/>
</antcall>
</target>
<target name="octane-deltablue-rhino" depends="jar">
<antcall target="run-octane-rhino">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/deltablue.js"/>
+ <param name="octane-tests" value="deltablue"/>
</antcall>
</target>
+
<!-- earley-boyer -->
<target name="octane-earley-boyer" depends="jar">
<antcall target="run-octane">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/earley-boyer.js"/>
+ <param name="octane-tests" value="earley-boyer"/>
</antcall>
</target>
<target name="octane-earley-boyer-v8" depends="jar">
<antcall target="run-octane-v8">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/earley-boyer.js"/>
+ <param name="octane-tests" value="earley-boyer"/>
</antcall>
</target>
<target name="octane-earley-boyer-rhino" depends="jar">
<antcall target="run-octane-rhino">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/earley-boyer.js"/>
+ <param name="octane-tests" value="earley-boyer"/>
</antcall>
</target>
+
<!-- gbemu -->
<target name="octane-gbemu" depends="jar">
<antcall target="run-octane">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/gbemu.js"/>
+ <param name="octane-tests" value="gbemu"/>
</antcall>
</target>
<target name="octane-gbemu-v8" depends="jar">
<antcall target="run-octane-v8">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/gbemu.js"/>
+ <param name="octane-tests" value="gbemu"/>
</antcall>
</target>
<target name="octane-gbemu-rhino" depends="jar">
<antcall target="run-octane-rhino">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/gbemu.js"/>
+ <param name="octane-tests" value="gbemu"/>
</antcall>
</target>
+
<!-- mandreel -->
<target name="octane-mandreel" depends="jar">
<antcall target="run-octane">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/mandreel.js"/>
+ <param name="octane-tests" value="mandreel"/>
</antcall>
</target>
<target name="octane-mandreel-v8" depends="jar">
<antcall target="run-octane-v8">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/mandreel.js"/>
+ <param name="octane-tests" value="mandreel"/>
</antcall>
</target>
<target name="octane-mandreel-rhino" depends="jar">
<antcall target="run-octane-rhino">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/mandreel.js"/>
+ <param name="octane-tests" value="mandreel"/>
</antcall>
</target>
+
<!-- navier-stokes -->
<target name="octane-navier-stokes" depends="jar">
<antcall target="run-octane">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/navier-stokes.js"/>
+ <param name="octane-tests" value="navier-stokes"/>
</antcall>
</target>
<target name="octane-navier-stokes-v8" depends="jar">
<antcall target="run-octane-v8">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/navier-stokes.js"/>
+ <param name="octane-tests" value="navier-stokes"/>
</antcall>
</target>
<target name="octane-navier-stokes-rhino" depends="jar">
<antcall target="run-octane-rhino">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/navier-stokes.js"/>
+ <param name="octane-tests" value="navier-stokes"/>
</antcall>
</target>
+
<!-- pdfjs -->
<target name="octane-pdfjs" depends="jar">
<antcall target="run-octane">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/pdfjs.js"/>
+ <param name="octane-tests" value="pdfjs"/>
</antcall>
</target>
<target name="octane-pdfjs-v8" depends="jar">
<antcall target="run-octane-v8">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/pdfjs.js"/>
+ <param name="octane-tests" value="pdfjs"/>
</antcall>
</target>
<target name="octane-pdfjs-rhino" depends="jar">
<antcall target="run-octane-rhino">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/pdfjs.js"/>
+ <param name="octane-tests" value="pdfjs"/>
</antcall>
</target>
+
<!-- raytrace -->
<target name="octane-raytrace" depends="jar">
<antcall target="run-octane">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/raytrace.js"/>
+ <param name="octane-tests" value="raytrace"/>
</antcall>
</target>
<target name="octane-raytrace-v8" depends="jar">
<antcall target="run-octane-v8">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/raytrace.js"/>
+ <param name="octane-tests" value="raytrace"/>
</antcall>
</target>
<target name="octane-raytrace-rhino" depends="jar">
<antcall target="run-octane-rhino">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/raytrace.js"/>
+ <param name="octane-tests" value="raytrace"/>
</antcall>
</target>
+
<!-- regexp -->
<target name="octane-regexp" depends="jar">
<antcall target="run-octane">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/regexp.js"/>
+ <param name="octane-tests" value="regexp"/>
</antcall>
</target>
<target name="octane-regexp-octane-v8" depends="jar">
<antcall target="run-octane-v8">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/regexp.js"/>
+ <param name="octane-tests" value="regexp"/>
</antcall>
</target>
<target name="octane-regexp-rhino" depends="jar">
<antcall target="run-octane-rhino">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/regexp.js"/>
+ <param name="octane-tests" value="regexp"/>
</antcall>
</target>
+
<!-- richards -->
<target name="octane-richards" depends="jar">
<antcall target="run-octane">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/richards.js"/>
+ <param name="octane-tests" value="richards"/>
</antcall>
</target>
<target name="octane-richards-v8" depends="jar">
<antcall target="run-octane-v8">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/richards.js"/>
+ <param name="octane-tests" value="richards"/>
</antcall>
</target>
<target name="octane-richards-rhino" depends="jar">
<antcall target="run-octane-rhino">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/richards.js"/>
+ <param name="octane-tests" value="richards"/>
</antcall>
</target>
+
<!-- splay -->
<target name="octane-splay" depends="jar">
<antcall target="run-octane">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/splay.js"/>
+ <param name="octane-tests" value="splay"/>
</antcall>
</target>
<target name="octane-splay-v8" depends="jar">
<antcall target="run-octane-v8">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/splay.js"/>
+ <param name="octane-tests" value="splay"/>
</antcall>
</target>
<target name="octane-splay-rhino" depends="jar">
<antcall target="run-octane-rhino">
- <param name="octane-tests" value="${octane-test-sys-prop.test.js.roots}/splay.js"/>
+ <param name="octane-tests" value="splay"/>
</antcall>
</target>
--- a/nashorn/src/jdk/nashorn/internal/runtime/AccessorProperty.java Fri May 24 23:27:52 2013 +0530
+++ b/nashorn/src/jdk/nashorn/internal/runtime/AccessorProperty.java Mon May 27 13:11:13 2013 +0200
@@ -306,14 +306,14 @@
protected Object getObjectValue(final ScriptObject self, final ScriptObject owner) {
if (isSpill()) {
return self.spill[getSlot()];
- } else {
- try {
- return getGetter(Object.class).invokeExact((Object)self);
- } catch (final Error|RuntimeException e) {
- throw e;
- } catch (final Throwable e) {
- throw new RuntimeException(e);
- }
+ }
+
+ try {
+ return getGetter(Object.class).invokeExact((Object)self);
+ } catch (final Error|RuntimeException e) {
+ throw e;
+ } catch (final Throwable e) {
+ throw new RuntimeException(e);
}
}
--- a/nashorn/src/jdk/nashorn/internal/runtime/Property.java Fri May 24 23:27:52 2013 +0530
+++ b/nashorn/src/jdk/nashorn/internal/runtime/Property.java Mon May 27 13:11:13 2013 +0200
@@ -52,6 +52,7 @@
* we can use leave flag byte initialized with (the default) zero value.
*/
+ /** Mask for property being both writable, enumerable and configurable */
public static final int WRITABLE_ENUMERABLE_CONFIGURABLE = 0b0000_0000_0000;
/** ECMA 8.6.1 - Is this property not writable? */
@@ -360,6 +361,7 @@
* @param self the this object
* @param owner the owner object
* @param value the new property value
+ * @param strict is this a strict setter?
*/
protected abstract void setObjectValue(ScriptObject self, ScriptObject owner, Object value, boolean strict);
--- a/nashorn/src/jdk/nashorn/internal/runtime/UserAccessorProperty.java Fri May 24 23:27:52 2013 +0530
+++ b/nashorn/src/jdk/nashorn/internal/runtime/UserAccessorProperty.java Mon May 27 13:11:13 2013 +0200
@@ -193,7 +193,6 @@
// getter/setter may be inherited. If so, proto is bound during lookup. In either
// inherited or self case, slot is also bound during lookup. Actual ScriptFunction
// to be called is retrieved everytime and applied.
- @SuppressWarnings("unused")
static Object userAccessorGetter(final ScriptObject proto, final int slot, final Object self) {
final ScriptObject container = (proto != null) ? proto : (ScriptObject)self;
final Object func = container.getSpill(slot);
@@ -211,7 +210,6 @@
return UNDEFINED;
}
- @SuppressWarnings("unused")
static void userAccessorSetter(final ScriptObject proto, final int slot, final String name, final Object self, final Object value) {
final ScriptObject container = (proto != null) ? proto : (ScriptObject)self;
final Object func = container.getSpill(slot);
--- a/nashorn/test/script/basic/compile-octane.js.EXPECTED Fri May 24 23:27:52 2013 +0530
+++ b/nashorn/test/script/basic/compile-octane.js.EXPECTED Mon May 27 13:11:13 2013 +0200
@@ -1,39 +1,13 @@
-Compiling... box2d.js
-Compiled OK: box2d.js
-
-Compiling... code-load.js
-Compiled OK: code-load.js
-
-Compiling... crypto.js
-Compiled OK: crypto.js
-
-Compiling... deltablue.js
-Compiled OK: deltablue.js
-
-Compiling... earley-boyer.js
-Compiled OK: earley-boyer.js
-
-Compiling... gbemu.js
-Compiled OK: gbemu.js
-
-Compiling... mandreel.js
-Compiled OK: mandreel.js
-
-Compiling... navier-stokes.js
-Compiled OK: navier-stokes.js
-
-Compiling... pdfjs.js
-Compiled OK: pdfjs.js
-
-Compiling... raytrace.js
-Compiled OK: raytrace.js
-
-Compiling... regexp.js
-Compiled OK: regexp.js
-
-Compiling... richards.js
-Compiled OK: richards.js
-
-Compiling... splay.js
-Compiled OK: splay.js
-
+Compiled OK: box2d
+Compiled OK: code-load
+Compiled OK: crypto
+Compiled OK: deltablue
+Compiled OK: earley-boyer
+Compiled OK: gbemu
+Compiled OK: mandreel
+Compiled OK: navier-stokes
+Compiled OK: pdfjs
+Compiled OK: raytrace
+Compiled OK: regexp
+Compiled OK: richards
+Compiled OK: splay
--- a/nashorn/test/script/basic/run-octane.js Fri May 24 23:27:52 2013 +0530
+++ b/nashorn/test/script/basic/run-octane.js Mon May 27 13:11:13 2013 +0200
@@ -26,36 +26,20 @@
*/
var tests = [
- "box2d.js",
- "code-load.js",
- "crypto.js",
- "deltablue.js",
- "earley-boyer.js",
- "gbemu.js",
- "mandreel.js",
- "navier-stokes.js",
- "pdfjs.js",
- "raytrace.js",
- "regexp.js",
- "richards.js",
- "splay.js"
+ {file:"box2d",suite:"Box2DBenchmark"},
+ {file:"code-load",suite:"CodeLoad"},
+ {file:"crypto",suite:"Crypto"},
+ {file:"deltablue",suite:"DeltaBlue"},
+ {file:"earley-boyer", suite:"EarleyBoyer"},
+ {file:"gbemu", suite:"GameboyBenchmark"},
+ {file:"mandreel", suite:"MandreelBenchmark"},
+ {file:"navier-stokes", suite:"NavierStokes"},
+ {file:"pdfjs", suite:"PdfJS"},
+ {file:"raytrace", suite:"RayTrace"},
+ {file:"regexp", suite:"RegExpSuite"},
+ {file:"richards", suite:"Richards"},
+ {file:"splay", suite:"Splay"}
];
-
-// hack, teardown breaks things defined in the global space, making it impossible
-// to do multiple consecutive benchmark runs with the same harness. I think it's a bug
-// that the setup and teardown aren't each others constructor and destructor but rather
-// that the benchmarks rely on partial global state. For shame, Octane!
-var ignoreTeardown = [
- { name: "box2d.js" },
- { name: "gbemu.js" },
-];
-
-
-//TODO mandreel can be compiled as a test, but not run multiple times unless modified to not have global state
-var compileOnly = {
- "mandreel.js" : true
-};
-
var dir = (typeof(__DIR__) == 'undefined') ? "test/script/basic/" : __DIR__;
// TODO: why is this path hard coded when it's defined in project properties?
@@ -71,110 +55,96 @@
}
function should_compile_only(name) {
- return (typeof compile_only !== 'undefined') || compileOnly[name] === true;
+ return (typeof compile_only !== 'undefined')
}
function run_one_benchmark(arg, iters) {
-
var file_name;
- var file = arg.split('/');
- if (file.length == 1) {
- file = arg.split('\\');
- }
-
- //trim off trailing path separators
- while (file[file.length - 1].indexOf(".js") == -1) {
- file.pop();
- }
- file_name = file[file.length - 1];
-
+ var file = (arg.file + ".js").split('/');
+
+ file_name = path + file[file.length - 1];
+
var compile_and_return = should_compile_only(file_name);
if (compile_and_return) {
if (typeof compile_only === 'undefined') { //for a run, skip compile onlies, don't even compile them
return;
}
- print("Compiling... " + file_name);
}
-
- load(path + 'base.js');
- load(arg);
+
+ print_verbose("Loading... " + file_name);
+ load(file_name);
if (compile_and_return) {
- print("Compiled OK: " + file_name);
- print("");
+ print_always("Compiled OK: " + arg.file);
return;
}
var success = true;
- var hiscore = 0;
- var loscore = 10e8;
var current_name;
- function PrintResult(name, result) {
- current_name = name;
- }
-
- function PrintError(name, error) {
- current_name = name;
- PrintResult(name, error);
- success = false;
- }
-
- function PrintScore(score) {
- if (success) {
- if (+score >= hiscore) {
- hiscore = +score;
- }
- if (+score <= loscore) {
- loscore = +score;
- }
- }
-
- if (verbose) {
- print("Score: " + score);
- }
- }
-
if (iters == undefined) {
iters = numberOfIterations;
} else {
numberOfIterations = iters;
}
-
- print(runtime + ": running " + file_name + "...");
-
- for (var i = 0; i < numberOfIterations; i++) {
- var callbacks =
- { NotifyResult: PrintResult,
- NotifyError: PrintError,
- NotifyScore: PrintScore };
-
- for (j in ignoreTeardown) {
- var ignore = ignoreTeardown[j];
- if (endsWith(arg, ignore.name)) {
- var teardownOverride = ignore.teardown;
- if (!teardownOverride) {
- teardownOverride = function() {};
- }
+
+ var benchmarks = eval(arg.suite + ".benchmarks");
+ for (var x = 0; x < benchmarks.length ; x++) {
+ benchmarks[x].Setup();
+ }
+ print_verbose("Running '" + arg.file + "' for " + iters + " iterations of no less than " + min_time + " seconds (" + runtime + ")");
+
+ var scores = [];
+
+ var min_time_ms = min_time * 1000;
+ var len = benchmarks.length;
+
+ for (var it = 0; it < iters + 1; it++) {
+ //every iteration must take a minimum of 10 secs
+ var ops = 0;
+ var elapsed = 0;
+ var start = new Date;
+ do {
+ for (var i = 0; i < len; i++) {
+ benchmarks[i].run();
+ }
+ ops += len;
+ elapsed = new Date - start;
+ } while (elapsed < min_time * 1000);
+
+ var score = ops / elapsed * 1000 * 60;
+ scores.push(score);
+ var name = it == 0 ? "warmup" : "iteration " + it;
+ print_verbose("[" + arg.file + "] " + name + " finished " + score.toFixed(0) + " ops/minute");
+ }
+ for (var x = 0; x < benchmarks.length ; x++) {
+ benchmarks[x].TearDown();
+ }
- for (k in BenchmarkSuite.suites) {
- var benchmarks = BenchmarkSuite.suites[k].benchmarks;
- for (l in benchmarks) {
- benchmarks[l].TearDown = teardownOverride;
- }
- }
- break;
- }
- }
-
- BenchmarkSuite.RunSuites(callbacks);
+ var min_score = 1e9;
+ var max_score = 0;
+ var mean_score = 0;
+ for (var x = 1; x < iters + 1 ; x++) {
+ mean_score += scores[x];
+ min_score = Math.min(min_score, scores[x]);
+ max_score = Math.max(max_score, scores[x]);
}
-
- var start = "Score: ";
- if (runtime != "") {
- start = runtime + ": ";
- }
- print(start + current_name + ' (version ' + BenchmarkSuite.version + '): ' + loscore + '-' + hiscore);
+ mean_score /= iters;
+ var res = "[" + arg.file + "] " + mean_score.toFixed(0);
+ if (verbose) {
+ res += " ops/minute (" + min_score.toFixed(0) + "-" + max_score.toFixed(0) + "), warmup=" + scores[0].toFixed(0);
+ }
+ print_always(res);
+}
+
+function print_always(x) {
+ print(x);
+}
+
+function print_verbose(x) {
+ if (verbose) {
+ print(x);
+ }
}
function run_suite(tests, iters) {
@@ -211,6 +181,7 @@
var tests_found = [];
var iters = undefined;
+var min_time = 5;
for (var i = 0; i < args.length; i++) {
arg = args[i];
@@ -220,21 +191,41 @@
runtime = args[++i];
} else if (arg == "--verbose") {
verbose = true;
+ } else if (arg == "--min-time") {
+ min_time = +args[++i];
} else if (arg == "") {
continue; //skip
} else {
- tests_found.push(arg);
+ var found = false;
+ for (j in tests) {
+ if (tests[j].file === arg) {
+ tests_found.push(tests[j]);
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ var str = "unknown test name: '" + arg + "' -- valid names are: ";
+ for (j in tests) {
+ if (j != 0) {
+ str += ", ";
+ }
+ str += "'" + tests[j].file + "'";
+ }
+ throw str;
+ }
}
}
if (tests_found.length == 0) {
for (i in tests) {
- tests_found.push(path + tests[i]);
+ tests_found.push(tests[i]);
}
}
tests_found.sort();
+load(path + 'base.js');
run_suite(tests_found, iters);