From 3ff0a44a1c62b9d3eb11c5c061c1a0f2bbdcd724 Mon Sep 17 00:00:00 2001 From: John-David Dalton Date: Mon, 22 Apr 2013 08:33:01 -0700 Subject: [PATCH] Tweak perf suite to use geometric mean and remove tests with high variability. Former-commit-id: ce4ff88ee6be007b761bcf991c59f30f28880973 --- perf/perf.js | 157 +++++++++++++++---------------- vendor/benchmark.js/README.md | 2 +- vendor/benchmark.js/benchmark.js | 132 +++++++++++++++----------- 3 files changed, 155 insertions(+), 136 deletions(-) diff --git a/perf/perf.js b/perf/perf.js index 8c187e095..5e827dc21 100644 --- a/perf/perf.js +++ b/perf/perf.js @@ -58,7 +58,7 @@ var reSpecialChars = /[.*+?^=!:${}()|[\]\/\\]/g; /** Used to score performance */ - var score = { 'a': 0, 'b': 0 }; + var score = { 'a': [], 'b': [] }; /** Used to queue benchmark suites */ var suites = []; @@ -105,6 +105,20 @@ : result.replace(RegExp(extension.replace(reSpecialChars, '\\$&') + '$'), ''); } + /** + * Computes the geometric mean (log-average) of an array of values. + * See http://en.wikipedia.org/wiki/Geometric_mean#Relationship_with_arithmetic_mean_of_logarithms. + * + * @private + * @param {Array} array The array of values. + * @returns {Number} The geometric mean. + */ + function getGeometricMean(array) { + return Math.pow(Math.E, _.reduce(array, function(sum, x) { + return sum + Math.log(x); + }, 0) / array.length) || 0; + } + /** * Gets the Hz, i.e. operations per second, of `bench` adjusted for the * margin of error. @@ -174,14 +188,6 @@ log(event.target); }, 'onComplete': function() { - var formatNumber = Benchmark.formatNumber, - fastest = this.filter('fastest'), - fastestHz = getHz(fastest[0]), - slowest = this.filter('slowest'), - slowestHz = getHz(slowest[0]), - aHz = getHz(this[0]), - bHz = getHz(this[1]); - for (var index = 0, length = this.length; index < length; index++) { var bench = this[index]; if (bench.error) { @@ -190,6 +196,14 @@ } } if (!errored) { + var formatNumber = Benchmark.formatNumber, + fastest = this.filter('fastest'), + fastestHz = getHz(fastest[0]), + slowest = this.filter('slowest'), + slowestHz = getHz(slowest[0]), + aHz = getHz(this[0]), + bHz = getHz(this[1]); + if (fastest.length > 1) { log('It\'s too close to call.'); aHz = bHz = slowestHz; @@ -204,8 +218,8 @@ ); } // add score adjusted for margin of error - score.a += aHz; - score.b += bHz; + score.a.push(aHz); + score.b.push(bHz); } // remove current suite from queue suites.shift(); @@ -215,14 +229,16 @@ suites[0].run({ 'async': !isJava }); } else { - var fastestTotalHz = Math.max(score.a, score.b), - slowestTotalHz = Math.min(score.a, score.b), - totalPercent = formatNumber(Math.round(((fastestTotalHz / slowestTotalHz) - 1) * 100)), - totalX = fastestTotalHz / slowestTotalHz, - message = 'is ' + totalPercent + '% ' + (totalX == 1 ? '' : '(' + formatNumber(totalX.toFixed(2)) + 'x) ') + 'faster than'; + var aMeanHz = getGeometricMean(score.a), + bMeanHz = getGeometricMean(score.b), + fastestMeanHz = Math.max(aMeanHz, bMeanHz), + slowestMeanHz = Math.min(aMeanHz, bMeanHz), + xFaster = fastestMeanHz / slowestMeanHz, + percentFaster = formatNumber(Math.round((xFaster - 1) * 100)), + message = 'is ' + percentFaster + '% ' + (xFaster == 1 ? '' : '(' + formatNumber(xFaster.toFixed(2)) + 'x) ') + 'faster than'; // report results - if (score.a >= score.b) { + if (aMeanHz >= bMeanHz) { log('\n' + buildName + ' ' + message + ' ' + otherName + '.'); } else { log('\n' + otherName + ' ' + message + ' ' + buildName + '.'); @@ -273,13 +289,14 @@ }\ \ if (typeof bindAll != "undefined") {\ - var bindAllObjects = Array(this.count),\ - funcNames = belt.functions(lodash);\ + var bindAllCount = -1,\ + bindAllObjects = Array(this.count),\ + funcNames = belt.functions(belt).slice(0, 40);\ \ // potentially expensive\n\ for (index = 0; index < this.count; index++) {\ bindAllObjects[index] = belt.reduce(funcNames, function(object, funcName) {\ - object[funcName] = lodash[funcName];\ + object[funcName] = belt[funcName];\ return object;\ }, {});\ }\ @@ -624,11 +641,11 @@ suites.push( Benchmark.Suite('`_.bindAll` iterating arguments') .add(buildName, { - 'fn': 'lodash.bindAll.apply(lodash, [bindAllObjects.pop()].concat(funcNames))', + 'fn': 'lodash.bindAll.apply(lodash, [bindAllObjects[++bindAllCount]].concat(funcNames))', 'teardown': 'function bindAll(){}' }) .add(otherName, { - 'fn': '_.bindAll.apply(_, [bindAllObjects.pop()].concat(funcNames))', + 'fn': '_.bindAll.apply(_, [bindAllObjects[++bindAllCount]].concat(funcNames))', 'teardown': 'function bindAll(){}' }) ); @@ -636,11 +653,11 @@ suites.push( Benchmark.Suite('`_.bindAll` iterating the `object`') .add(buildName, { - 'fn': 'lodash.bindAll(bindAllObjects.pop())', + 'fn': 'lodash.bindAll(bindAllObjects[++bindAllCount])', 'teardown': 'function bindAll(){}' }) .add(otherName, { - 'fn': '_.bindAll(bindAllObjects.pop())', + 'fn': '_.bindAll(bindAllObjects[++bindAllCount])', 'teardown': 'function bindAll(){}' }) ); @@ -1032,22 +1049,14 @@ suites.push( Benchmark.Suite('`_.indexOf`') - .add(buildName, '\ - lodash.indexOf(numbers, 9)' - ) - .add(otherName, '\ - _.indexOf(numbers, 9)' - ) - ); - - suites.push( - Benchmark.Suite('`_.indexOf` with `isSorted`') - .add(buildName, '\ - lodash.indexOf(numbers, 19, true)' - ) - .add(otherName, '\ - _.indexOf(numbers, 19, true)' - ) + .add(buildName, { + 'fn': 'lodash.indexOf(twoHundredValues, 199)', + 'teardown': 'function multiArrays(){}' + }) + .add(buildName, { + 'fn': '_.indexOf(twoHundredValues, 199)', + 'teardown': 'function multiArrays(){}' + }) ); /*--------------------------------------------------------------------------*/ @@ -1137,13 +1146,13 @@ .add(buildName, { 'fn': '\ lodash.isEqual(numbers, numbers2);\ - lodash.isEqual(twoNumbers, twoNumbers2);', + lodash.isEqual(twoNumbers, twoNumbers2)', 'teardown': 'function isEqual(){}' }) .add(otherName, { 'fn': '\ _.isEqual(numbers, numbers2);\ - _.isEqual(twoNumbers, twoNumbers2);', + _.isEqual(twoNumbers, twoNumbers2)', 'teardown': 'function isEqual(){}' }) ); @@ -1153,13 +1162,13 @@ .add(buildName, { 'fn': '\ lodash.isEqual(nestedNumbers, nestedNumbers2);\ - lodash.isEqual(nestedNumbers2, nestedNumbers3);', + lodash.isEqual(nestedNumbers2, nestedNumbers3)', 'teardown': 'function isEqual(){}' }) .add(otherName, { 'fn': '\ _.isEqual(nestedNumbers, nestedNumbers2);\ - _.isEqual(nestedNumbers2, nestedNumbers3);', + _.isEqual(nestedNumbers2, nestedNumbers3)', 'teardown': 'function isEqual(){}' }) ); @@ -1169,13 +1178,13 @@ .add(buildName, { 'fn': '\ lodash.isEqual(objects, objects2);\ - lodash.isEqual(simpleObjects, simpleObjects2);', + lodash.isEqual(simpleObjects, simpleObjects2)', 'teardown': 'function isEqual(){}' }) .add(otherName, { 'fn': '\ _.isEqual(objects, objects2);\ - _.isEqual(simpleObjects, simpleObjects2);', + _.isEqual(simpleObjects, simpleObjects2)', 'teardown': 'function isEqual(){}' }) ); @@ -1185,13 +1194,13 @@ .add(buildName, { 'fn': '\ lodash.isEqual(object, object2);\ - lodash.isEqual(simpleObject, simpleObject2);', + lodash.isEqual(simpleObject, simpleObject2)', 'teardown': 'function isEqual(){}' }) .add(otherName, { 'fn': '\ _.isEqual(object, object2);\ - _.isEqual(simpleObject, simpleObject2);', + _.isEqual(simpleObject, simpleObject2)', 'teardown': 'function isEqual(){}' }) ); @@ -1212,7 +1221,7 @@ lodash.isObject(object);\ lodash.isObject(1);\ lodash.isRegExp(regexp);\ - lodash.isRegExp(object);' + lodash.isRegExp(object)' ) .add(otherName, '\ _.isArguments(arguments);\ @@ -1226,7 +1235,7 @@ _.isObject(object);\ _.isObject(1);\ _.isRegExp(regexp);\ - _.isRegExp(object);' + _.isRegExp(object)' ) ); @@ -1390,13 +1399,13 @@ lodash.reduce(numbers, function(result, value, index) {\ result[index] = value;\ return result;\ - }, {});' + }, {})' ) .add(otherName, '\ _.reduce(numbers, function(result, value, index) {\ result[index] = value;\ return result;\ - }, {});' + }, {})' ) ); @@ -1406,13 +1415,13 @@ lodash.reduce(object, function(result, value, key) {\ result.push(key, value);\ return result;\ - }, []);' + }, [])' ) .add(otherName, '\ _.reduce(object, function(result, value, key) {\ result.push(key, value);\ return result;\ - }, []);' + }, [])' ) ); @@ -1424,13 +1433,13 @@ lodash.reduceRight(numbers, function(result, value, index) {\ result[index] = value;\ return result;\ - }, {});' + }, {})' ) .add(otherName, '\ _.reduceRight(numbers, function(result, value, index) {\ result[index] = value;\ return result;\ - }, {});' + }, {})' ) ); @@ -1440,13 +1449,13 @@ lodash.reduceRight(object, function(result, value, key) {\ result.push(key, value);\ return result;\ - }, []);' + }, [])' ) .add(otherName, '\ _.reduceRight(object, function(result, value, key) {\ result.push(key, value);\ return result;\ - }, []);' + }, [])' ) ); @@ -1598,16 +1607,6 @@ /*--------------------------------------------------------------------------*/ - suites.push( - Benchmark.Suite('`_.sortedIndex`') - .add(buildName, '\ - lodash.sortedIndex(numbers, 25)' - ) - .add(otherName, '\ - _.sortedIndex(numbers, 25)' - ) - ); - suites.push( Benchmark.Suite('`_.sortedIndex` with `callback`') .add(buildName, { @@ -1727,11 +1726,11 @@ suites.push( Benchmark.Suite('`_.union` iterating an array of 75 elements') .add(buildName, { - 'fn': 'lodash.union(fiftyValues, twentyFiveValues2);', + 'fn': 'lodash.union(fiftyValues, twentyFiveValues2)', 'teardown': 'function multiArrays(){}' }) .add(otherName, { - 'fn': '_.union(fiftyValues, twentyFiveValues2);', + 'fn': '_.union(fiftyValues, twentyFiveValues2)', 'teardown': 'function multiArrays(){}' }) ); @@ -1753,7 +1752,7 @@ .add(buildName, '\ lodash.uniq(numbers.concat(twoNumbers, fourNumbers), function(num) {\ return num % 2;\ - });' + })' ) .add(otherName, '\ _.uniq(numbers.concat(twoNumbers, fourNumbers), function(num) {\ @@ -1765,11 +1764,11 @@ suites.push( Benchmark.Suite('`_.uniq` iterating an array of 200 elements') .add(buildName, { - 'fn': 'lodash.uniq(oneHundredValues.concat(oneHundredValues2));', + 'fn': 'lodash.uniq(oneHundredValues.concat(oneHundredValues2))', 'teardown': 'function multiArrays(){}' }) .add(otherName, { - 'fn': '_.uniq(oneHundredValues.concat(oneHundredValues2));', + 'fn': '_.uniq(oneHundredValues.concat(oneHundredValues2))', 'teardown': 'function multiArrays(){}' }) ); @@ -1779,11 +1778,11 @@ suites.push( Benchmark.Suite('`_.unzip`') .add(buildName, { - 'fn': 'lodash.unzip(zipped);', + 'fn': 'lodash.unzip(zipped)', 'teardown': 'function zip(){}' }) .add(otherName, { - 'fn': '_.unzip(zipped);', + 'fn': '_.unzip(zipped)', 'teardown': 'function zip(){}' }) ); @@ -1805,11 +1804,11 @@ suites.push( Benchmark.Suite('`_.where`') .add(buildName, { - 'fn': 'lodash.where(objects, whereObject);', + 'fn': 'lodash.where(objects, whereObject)', 'teardown': 'function where(){}' }) .add(otherName, { - 'fn': '_.where(objects, whereObject);', + 'fn': '_.where(objects, whereObject)', 'teardown': 'function where(){}' }) ); @@ -1831,11 +1830,11 @@ suites.push( Benchmark.Suite('`_.zip`') .add(buildName, { - 'fn': 'lodash.zip.apply(lodash, unzipped);', + 'fn': 'lodash.zip.apply(lodash, unzipped)', 'teardown': 'function zip(){}' }) .add(otherName, { - 'fn': '_.zip.apply(_, unzipped);', + 'fn': '_.zip.apply(_, unzipped)', 'teardown': 'function zip(){}' }) ); diff --git a/vendor/benchmark.js/README.md b/vendor/benchmark.js/README.md index 21a077fcd..e0ccd18e0 100644 --- a/vendor/benchmark.js/README.md +++ b/vendor/benchmark.js/README.md @@ -100,7 +100,7 @@ suite.add('RegExp#test', function() { console.log(String(event.target)); }) .on('complete', function() { - console.log('Fastest is ' + _.pluck(this.filter('fastest'), 'name')); + console.log('Fastest is ' + this.filter('fastest').pluck('name')); }) // run async .run({ 'async': true }); diff --git a/vendor/benchmark.js/benchmark.js b/vendor/benchmark.js/benchmark.js index 00281b5d4..4a48ab9b5 100644 --- a/vendor/benchmark.js/benchmark.js +++ b/vendor/benchmark.js/benchmark.js @@ -30,6 +30,9 @@ /** Used to assign each benchmark an incrimented id */ var counter = 0; + /** Used to make every compiled test unique */ + var uidCounter = 0; + /** Used to detect primitive types */ var rePrimitive = /^(?:boolean|number|string|undefined)$/; @@ -139,6 +142,7 @@ max = Math.max, min = Math.min, pow = Math.pow, + push = arrayRef.push, setTimeout = context.setTimeout, shift = arrayRef.shift, slice = arrayRef.slice, @@ -1444,10 +1448,9 @@ // ...the z-stat is greater than 1.96 or less than -1.96 // http://www.statisticslectures.com/topics/mannwhitneyu/ zStat = getZ(u); - return abs(zStat) > 1.96 ? (zStat > 0 ? -1 : 1) : 0; + return abs(zStat) > 1.96 ? (u == u1 ? 1 : -1) : 0; } // ...the U value is less than or equal the critical U value - // http://www.geoib.com/mann-whitney-u-test.html critical = maxSize < 5 || minSize < 3 ? 0 : uTable[maxSize][minSize - 3]; return u <= critical ? (u == u1 ? 1 : -1) : 0; } @@ -1560,22 +1563,18 @@ function clock() { var applet, options = Benchmark.options, + templateData = {}, timers = [{ 'ns': timer.ns, 'res': max(0.0015, getRes('ms')), 'unit': 'ms' }]; - var templateData = { - 'begin': interpolate('s#=new n#'), - 'end': interpolate('r#=(new n#-s#)/1e3'), - 'uid': uid - }; - // lazy define for hi-res timers clock = function(clone) { var deferred; + templateData.uid = uid + uidCounter++; + if (clone instanceof Deferred) { deferred = clone; clone = deferred.benchmark; } - var bench = clone._original, fn = bench.fn, fnArg = deferred ? getFirstArgument(fn) || 'deferred' : '', @@ -1588,6 +1587,45 @@ 'teardown': getSource(bench.teardown, interpolate('m#.teardown()')) }); + // use API of chosen timer + if (timer.unit == 'ns') { + if (timer.ns.nanoTime) { + _.extend(templateData, { + 'begin': interpolate('s#=n#.nanoTime()'), + 'end': interpolate('r#=(n#.nanoTime()-s#)/1e9') + }); + } else { + _.extend(templateData, { + 'begin': interpolate('s#=n#()'), + 'end': interpolate('r#=n#(s#);r#=r#[0]+(r#[1]/1e9)') + }); + } + } + else if (timer.unit == 'us') { + if (timer.ns.stop) { + _.extend(templateData, { + 'begin': interpolate('s#=n#.start()'), + 'end': interpolate('r#=n#.microseconds()/1e6') + }); + } else if (perfName) { + _.extend(templateData, { + 'begin': interpolate('s#=n#.' + perfName + '()'), + 'end': interpolate('r#=(n#.' + perfName + '()-s#)/1e3') + }); + } else { + _.extend(templateData, { + 'begin': interpolate('s#=n#()'), + 'end': interpolate('r#=(n#()-s#)/1e6') + }); + } + } + else { + _.extend(templateData, { + 'begin': interpolate('s#=new n#'), + 'end': interpolate('r#=(new n#-s#)/1e3') + }); + } + var count = bench.count = clone.count, decompilable = support.decompilation || stringable, id = bench.id, @@ -1609,6 +1647,16 @@ ns = timer.ns = new applet.Packages.nano; } } + // define `timer` methods + timer.start = createFunction( + interpolate('o#'), + interpolate('var n#=this.ns,${begin};o#.elapsed=0;o#.timeStamp=s#') + ); + + timer.stop = createFunction( + interpolate('o#'), + interpolate('var n#=this.ns,s#=o#.timeStamp,${end};o#.elapsed=r#') + ); // Compile in setup/teardown functions and the test loop. // Create a new compiled test, instead of using the cached `bench.compiled`, @@ -1643,7 +1691,7 @@ // pretest to determine if compiled code is exits early, usually by a // rogue `return` statement, by checking for a return object with the uid bench.count = 1; - compiled = (compiled.call(bench, context, timer) || {}).uid == uid && compiled; + compiled = (compiled.call(bench, context, timer) || {}).uid == templateData.uid && compiled; bench.count = count; } } catch(e) { @@ -1762,7 +1810,7 @@ */ function interpolate(string) { // replaces all occurrences of `#` with a unique number and template tokens with content - return _.template(string.replace(/\#/g, /\d+/.exec(uid)), templateData || {}); + return _.template(string.replace(/\#/g, /\d+/.exec(templateData.uid)), templateData); } /*----------------------------------------------------------------------*/ @@ -1816,50 +1864,6 @@ if (timer.res == Infinity) { throw new Error('Benchmark.js was unable to find a working timer.'); } - // use API of chosen timer - if (timer.unit == 'ns') { - if (timer.ns.nanoTime) { - _.extend(templateData, { - 'begin': interpolate('s#=n#.nanoTime()'), - 'end': interpolate('r#=(n#.nanoTime()-s#)/1e9') - }); - } else { - _.extend(templateData, { - 'begin': interpolate('s#=n#()'), - 'end': interpolate('r#=n#(s#);r#=r#[0]+(r#[1]/1e9)') - }); - } - } - else if (timer.unit == 'us') { - if (timer.ns.stop) { - _.extend(templateData, { - 'begin': interpolate('s#=n#.start()'), - 'end': interpolate('r#=n#.microseconds()/1e6') - }); - } else if (perfName) { - _.extend(templateData, { - 'begin': interpolate('s#=n#.' + perfName + '()'), - 'end': interpolate('r#=(n#.' + perfName + '()-s#)/1e3') - }); - } else { - _.extend(templateData, { - 'begin': interpolate('s#=n#()'), - 'end': interpolate('r#=(n#()-s#)/1e6') - }); - } - } - - // define `timer` methods - timer.start = createFunction( - interpolate('o#'), - interpolate('var n#=this.ns,${begin};o#.elapsed=0;o#.timeStamp=s#') - ); - - timer.stop = createFunction( - interpolate('o#'), - interpolate('var n#=this.ns,s#=o#.timeStamp,${end};o#.elapsed=r#') - ); - // resolve time span required to achieve a percent uncertainty of at most 1% // http://spiff.rit.edu/classes/phys273/uncert/uncert.html options.minTime || (options.minTime = max(timer.res / 2 / 0.01, 0.05)); @@ -2356,6 +2360,11 @@ 'support': support }); + // Add Lo-Dash methods to Benchmark + _.each(['each', 'forEach', 'forOwn', 'has', 'indexOf', 'map', 'pluck', 'reduce'], function(methodName) { + Benchmark[methodName] = _[methodName]; + }); + /*------------------------------------------------------------------------*/ _.extend(Benchmark.prototype, { @@ -2777,11 +2786,12 @@ 'off': off, 'on': on, 'pop': arrayRef.pop, - 'push': arrayRef.push, + 'push': push, 'reset': resetSuite, 'run': runSuite, 'reverse': arrayRef.reverse, 'shift': shift, + 'slice': arrayRef.slice, 'sort': arrayRef.sort, 'splice': arrayRef.splice, 'unshift': arrayRef.unshift @@ -2798,6 +2808,16 @@ /*------------------------------------------------------------------------*/ + // add Lo-Dash methods as Suite methods + _.each(['each', 'forEach', 'indexOf', 'map', 'pluck', 'reduce'], function(methodName) { + var func = _[methodName]; + Suite.prototype[methodName] = function() { + var args = [this]; + push.apply(args, arguments); + return func.apply(_, args); + }; + }); + // avoid array-like object bugs with `Array#shift` and `Array#splice` // in Firefox < 10 and IE < 9 if (!_.support.spliceObjects) {