Tweak perf suite to use geometric mean and remove tests with high variability.

Former-commit-id: ce4ff88ee6be007b761bcf991c59f30f28880973

perf/perf.js

@@ -58,7 +58,7 @@
   var reSpecialChars = /[.*+?^=!:${}()|[\]\/\\]/g;
 
   /** Used to score performance */
-  var score = { 'a': 0, 'b': 0 };
+  var score = { 'a': [], 'b': [] };
 
   /** Used to queue benchmark suites */
   var suites = [];
@@ -105,6 +105,20 @@
       : result.replace(RegExp(extension.replace(reSpecialChars, '\\$&') + '$'), '');
   }
 
+  /**
+   * Computes the geometric mean (log-average) of an array of values.
+   * See http://en.wikipedia.org/wiki/Geometric_mean#Relationship_with_arithmetic_mean_of_logarithms.
+   *
+   * @private
+   * @param {Array} array The array of values.
+   * @returns {Number} The geometric mean.
+   */
+  function getGeometricMean(array) {
+    return Math.pow(Math.E, _.reduce(array, function(sum, x) {
+      return sum + Math.log(x);
+    }, 0) / array.length) || 0;
+  }
+
   /**
    * Gets the Hz, i.e. operations per second, of `bench` adjusted for the
    * margin of error.
@@ -174,14 +188,6 @@
       log(event.target);
     },
     'onComplete': function() {
-      var formatNumber = Benchmark.formatNumber,
-          fastest = this.filter('fastest'),
-          fastestHz = getHz(fastest[0]),
-          slowest = this.filter('slowest'),
-          slowestHz = getHz(slowest[0]),
-          aHz = getHz(this[0]),
-          bHz = getHz(this[1]);
-
       for (var index = 0, length = this.length; index < length; index++) {
         var bench = this[index];
         if (bench.error) {
@@ -190,6 +196,14 @@
         }
       }
       if (!errored) {
+        var formatNumber = Benchmark.formatNumber,
+            fastest = this.filter('fastest'),
+            fastestHz = getHz(fastest[0]),
+            slowest = this.filter('slowest'),
+            slowestHz = getHz(slowest[0]),
+            aHz = getHz(this[0]),
+            bHz = getHz(this[1]);
+
         if (fastest.length > 1) {
           log('It\'s too close to call.');
           aHz = bHz = slowestHz;
@@ -204,8 +218,8 @@
           );
         }
         // add score adjusted for margin of error
-        score.a += aHz;
+        score.a.push(aHz);
-        score.b += bHz;
+        score.b.push(bHz);
       }
       // remove current suite from queue
       suites.shift();
@@ -215,14 +229,16 @@
         suites[0].run({ 'async': !isJava });
       }
       else {
-        var fastestTotalHz = Math.max(score.a, score.b),
+        var aMeanHz = getGeometricMean(score.a),
-            slowestTotalHz = Math.min(score.a, score.b),
+            bMeanHz = getGeometricMean(score.b),
-            totalPercent = formatNumber(Math.round(((fastestTotalHz  / slowestTotalHz) - 1) * 100)),
+            fastestMeanHz = Math.max(aMeanHz, bMeanHz),
-            totalX = fastestTotalHz / slowestTotalHz,
+            slowestMeanHz = Math.min(aMeanHz, bMeanHz),
-            message = 'is ' + totalPercent + '% ' + (totalX == 1 ? '' : '(' + formatNumber(totalX.toFixed(2)) + 'x) ') + 'faster than';
+            xFaster = fastestMeanHz / slowestMeanHz,
+            percentFaster = formatNumber(Math.round((xFaster - 1) * 100)),
+            message = 'is ' + percentFaster + '% ' + (xFaster == 1 ? '' : '(' + formatNumber(xFaster.toFixed(2)) + 'x) ') + 'faster than';
 
         // report results
-        if (score.a >= score.b) {
+        if (aMeanHz >= bMeanHz) {
           log('\n' + buildName + ' ' + message + ' ' + otherName + '.');
         } else {
           log('\n' + otherName + ' ' + message + ' ' + buildName + '.');
@@ -273,13 +289,14 @@
       }\
       \
       if (typeof bindAll != "undefined") {\
-        var bindAllObjects = Array(this.count),\
+        var bindAllCount = -1,\
-            funcNames = belt.functions(lodash);\
+            bindAllObjects = Array(this.count),\
+            funcNames = belt.functions(belt).slice(0, 40);\
         \
         // potentially expensive\n\
         for (index = 0; index < this.count; index++) {\
           bindAllObjects[index] = belt.reduce(funcNames, function(object, funcName) {\
-            object[funcName] = lodash[funcName];\
+            object[funcName] = belt[funcName];\
             return object;\
           }, {});\
         }\
@@ -624,11 +641,11 @@
   suites.push(
     Benchmark.Suite('`_.bindAll` iterating arguments')
       .add(buildName, {
-        'fn': 'lodash.bindAll.apply(lodash, [bindAllObjects.pop()].concat(funcNames))',
+        'fn': 'lodash.bindAll.apply(lodash, [bindAllObjects[++bindAllCount]].concat(funcNames))',
         'teardown': 'function bindAll(){}'
       })
       .add(otherName, {
-        'fn': '_.bindAll.apply(_, [bindAllObjects.pop()].concat(funcNames))',
+        'fn': '_.bindAll.apply(_, [bindAllObjects[++bindAllCount]].concat(funcNames))',
         'teardown': 'function bindAll(){}'
       })
   );
@@ -636,11 +653,11 @@
   suites.push(
     Benchmark.Suite('`_.bindAll` iterating the `object`')
       .add(buildName, {
-        'fn': 'lodash.bindAll(bindAllObjects.pop())',
+        'fn': 'lodash.bindAll(bindAllObjects[++bindAllCount])',
         'teardown': 'function bindAll(){}'
       })
       .add(otherName, {
-        'fn': '_.bindAll(bindAllObjects.pop())',
+        'fn': '_.bindAll(bindAllObjects[++bindAllCount])',
         'teardown': 'function bindAll(){}'
       })
   );
@@ -1032,22 +1049,14 @@
 
   suites.push(
     Benchmark.Suite('`_.indexOf`')
-      .add(buildName, '\
+      .add(buildName, {
-        lodash.indexOf(numbers, 9)'
+        'fn': 'lodash.indexOf(twoHundredValues, 199)',
-      )
+        'teardown': 'function multiArrays(){}'
-      .add(otherName, '\
+      })
-        _.indexOf(numbers, 9)'
+      .add(buildName, {
-      )
+        'fn': '_.indexOf(twoHundredValues, 199)',
-  );
+        'teardown': 'function multiArrays(){}'
-
+      })
-  suites.push(
-    Benchmark.Suite('`_.indexOf` with `isSorted`')
-      .add(buildName, '\
-        lodash.indexOf(numbers, 19, true)'
-      )
-      .add(otherName, '\
-        _.indexOf(numbers, 19, true)'
-      )
   );
 
   /*--------------------------------------------------------------------------*/
@@ -1137,13 +1146,13 @@
       .add(buildName, {
         'fn': '\
           lodash.isEqual(numbers, numbers2);\
-          lodash.isEqual(twoNumbers, twoNumbers2);',
+          lodash.isEqual(twoNumbers, twoNumbers2)',
         'teardown': 'function isEqual(){}'
       })
       .add(otherName, {
         'fn': '\
           _.isEqual(numbers, numbers2);\
-          _.isEqual(twoNumbers, twoNumbers2);',
+          _.isEqual(twoNumbers, twoNumbers2)',
         'teardown': 'function isEqual(){}'
       })
   );
@@ -1153,13 +1162,13 @@
       .add(buildName, {
         'fn': '\
           lodash.isEqual(nestedNumbers, nestedNumbers2);\
-          lodash.isEqual(nestedNumbers2, nestedNumbers3);',
+          lodash.isEqual(nestedNumbers2, nestedNumbers3)',
         'teardown': 'function isEqual(){}'
       })
       .add(otherName, {
         'fn': '\
           _.isEqual(nestedNumbers, nestedNumbers2);\
-          _.isEqual(nestedNumbers2, nestedNumbers3);',
+          _.isEqual(nestedNumbers2, nestedNumbers3)',
         'teardown': 'function isEqual(){}'
       })
   );
@@ -1169,13 +1178,13 @@
       .add(buildName, {
         'fn': '\
           lodash.isEqual(objects, objects2);\
-          lodash.isEqual(simpleObjects, simpleObjects2);',
+          lodash.isEqual(simpleObjects, simpleObjects2)',
         'teardown': 'function isEqual(){}'
       })
       .add(otherName, {
         'fn': '\
           _.isEqual(objects, objects2);\
-          _.isEqual(simpleObjects, simpleObjects2);',
+          _.isEqual(simpleObjects, simpleObjects2)',
         'teardown': 'function isEqual(){}'
       })
   );
@@ -1185,13 +1194,13 @@
       .add(buildName, {
         'fn': '\
           lodash.isEqual(object, object2);\
-          lodash.isEqual(simpleObject, simpleObject2);',
+          lodash.isEqual(simpleObject, simpleObject2)',
         'teardown': 'function isEqual(){}'
       })
       .add(otherName, {
         'fn': '\
           _.isEqual(object, object2);\
-          _.isEqual(simpleObject, simpleObject2);',
+          _.isEqual(simpleObject, simpleObject2)',
         'teardown': 'function isEqual(){}'
       })
   );
@@ -1212,7 +1221,7 @@
         lodash.isObject(object);\
         lodash.isObject(1);\
         lodash.isRegExp(regexp);\
-        lodash.isRegExp(object);'
+        lodash.isRegExp(object)'
       )
       .add(otherName, '\
         _.isArguments(arguments);\
@@ -1226,7 +1235,7 @@
         _.isObject(object);\
         _.isObject(1);\
         _.isRegExp(regexp);\
-        _.isRegExp(object);'
+        _.isRegExp(object)'
       )
   );
 
@@ -1390,13 +1399,13 @@
         lodash.reduce(numbers, function(result, value, index) {\
           result[index] = value;\
           return result;\
-        }, {});'
+        }, {})'
       )
       .add(otherName, '\
         _.reduce(numbers, function(result, value, index) {\
           result[index] = value;\
           return result;\
-        }, {});'
+        }, {})'
       )
   );
 
@@ -1406,13 +1415,13 @@
         lodash.reduce(object, function(result, value, key) {\
           result.push(key, value);\
           return result;\
-        }, []);'
+        }, [])'
       )
       .add(otherName, '\
         _.reduce(object, function(result, value, key) {\
           result.push(key, value);\
           return result;\
-        }, []);'
+        }, [])'
       )
   );
 
@@ -1424,13 +1433,13 @@
         lodash.reduceRight(numbers, function(result, value, index) {\
           result[index] = value;\
           return result;\
-        }, {});'
+        }, {})'
       )
       .add(otherName, '\
         _.reduceRight(numbers, function(result, value, index) {\
           result[index] = value;\
           return result;\
-        }, {});'
+        }, {})'
       )
   );
 
@@ -1440,13 +1449,13 @@
         lodash.reduceRight(object, function(result, value, key) {\
           result.push(key, value);\
           return result;\
-        }, []);'
+        }, [])'
       )
       .add(otherName, '\
         _.reduceRight(object, function(result, value, key) {\
           result.push(key, value);\
           return result;\
-        }, []);'
+        }, [])'
       )
   );
 
@@ -1598,16 +1607,6 @@
 
   /*--------------------------------------------------------------------------*/
 
-  suites.push(
-    Benchmark.Suite('`_.sortedIndex`')
-      .add(buildName, '\
-        lodash.sortedIndex(numbers, 25)'
-      )
-      .add(otherName, '\
-        _.sortedIndex(numbers, 25)'
-      )
-  );
-
   suites.push(
     Benchmark.Suite('`_.sortedIndex` with `callback`')
       .add(buildName, {
@@ -1727,11 +1726,11 @@
   suites.push(
     Benchmark.Suite('`_.union` iterating an array of 75 elements')
       .add(buildName, {
-        'fn': 'lodash.union(fiftyValues, twentyFiveValues2);',
+        'fn': 'lodash.union(fiftyValues, twentyFiveValues2)',
         'teardown': 'function multiArrays(){}'
       })
       .add(otherName, {
-        'fn': '_.union(fiftyValues, twentyFiveValues2);',
+        'fn': '_.union(fiftyValues, twentyFiveValues2)',
         'teardown': 'function multiArrays(){}'
       })
   );
@@ -1753,7 +1752,7 @@
       .add(buildName, '\
         lodash.uniq(numbers.concat(twoNumbers, fourNumbers), function(num) {\
           return num % 2;\
-        });'
+        })'
       )
       .add(otherName, '\
         _.uniq(numbers.concat(twoNumbers, fourNumbers), function(num) {\
@@ -1765,11 +1764,11 @@
   suites.push(
     Benchmark.Suite('`_.uniq` iterating an array of 200 elements')
       .add(buildName, {
-        'fn': 'lodash.uniq(oneHundredValues.concat(oneHundredValues2));',
+        'fn': 'lodash.uniq(oneHundredValues.concat(oneHundredValues2))',
         'teardown': 'function multiArrays(){}'
       })
       .add(otherName, {
-        'fn': '_.uniq(oneHundredValues.concat(oneHundredValues2));',
+        'fn': '_.uniq(oneHundredValues.concat(oneHundredValues2))',
         'teardown': 'function multiArrays(){}'
       })
   );
@@ -1779,11 +1778,11 @@
   suites.push(
     Benchmark.Suite('`_.unzip`')
       .add(buildName, {
-        'fn': 'lodash.unzip(zipped);',
+        'fn': 'lodash.unzip(zipped)',
         'teardown': 'function zip(){}'
       })
       .add(otherName, {
-        'fn': '_.unzip(zipped);',
+        'fn': '_.unzip(zipped)',
         'teardown': 'function zip(){}'
       })
   );
@@ -1805,11 +1804,11 @@
   suites.push(
     Benchmark.Suite('`_.where`')
       .add(buildName, {
-        'fn': 'lodash.where(objects, whereObject);',
+        'fn': 'lodash.where(objects, whereObject)',
         'teardown': 'function where(){}'
       })
       .add(otherName, {
-        'fn': '_.where(objects, whereObject);',
+        'fn': '_.where(objects, whereObject)',
         'teardown': 'function where(){}'
       })
   );
@@ -1831,11 +1830,11 @@
   suites.push(
     Benchmark.Suite('`_.zip`')
       .add(buildName, {
-        'fn': 'lodash.zip.apply(lodash, unzipped);',
+        'fn': 'lodash.zip.apply(lodash, unzipped)',
         'teardown': 'function zip(){}'
       })
       .add(otherName, {
-        'fn': '_.zip.apply(_, unzipped);',
+        'fn': '_.zip.apply(_, unzipped)',
         'teardown': 'function zip(){}'
       })
   );

vendor/benchmark.js/README.md

@@ -100,7 +100,7 @@ suite.add('RegExp#test', function() {
   console.log(String(event.target));
 })
 .on('complete', function() {
-  console.log('Fastest is ' + _.pluck(this.filter('fastest'), 'name'));
+  console.log('Fastest is ' + this.filter('fastest').pluck('name'));
 })
 // run async
 .run({ 'async': true });

vendor/benchmark.js/benchmark.js

@@ -30,6 +30,9 @@
   /** Used to assign each benchmark an incrimented id */
   var counter = 0;
 
+  /** Used to make every compiled test unique */
+  var uidCounter = 0;
+
   /** Used to detect primitive types */
   var rePrimitive = /^(?:boolean|number|string|undefined)$/;
 
@@ -139,6 +142,7 @@
         max = Math.max,
         min = Math.min,
         pow = Math.pow,
+        push = arrayRef.push,
         setTimeout = context.setTimeout,
         shift = arrayRef.shift,
         slice = arrayRef.slice,
@@ -1444,10 +1448,9 @@
         // ...the z-stat is greater than 1.96 or less than -1.96
         // http://www.statisticslectures.com/topics/mannwhitneyu/
         zStat = getZ(u);
-        return abs(zStat) > 1.96 ? (zStat > 0 ? -1 : 1) : 0;
+        return abs(zStat) > 1.96 ? (u == u1 ? 1 : -1) : 0;
       }
       // ...the U value is less than or equal the critical U value
-      // http://www.geoib.com/mann-whitney-u-test.html
       critical = maxSize < 5 || minSize < 3 ? 0 : uTable[maxSize][minSize - 3];
       return u <= critical ? (u == u1 ? 1 : -1) : 0;
     }
@@ -1560,22 +1563,18 @@
     function clock() {
       var applet,
           options = Benchmark.options,
+          templateData = {},
           timers = [{ 'ns': timer.ns, 'res': max(0.0015, getRes('ms')), 'unit': 'ms' }];
 
-      var templateData = {
-        'begin': interpolate('s#=new n#'),
-        'end': interpolate('r#=(new n#-s#)/1e3'),
-        'uid': uid
-      };
-
       // lazy define for hi-res timers
       clock = function(clone) {
         var deferred;
+        templateData.uid = uid + uidCounter++;
+
         if (clone instanceof Deferred) {
           deferred = clone;
           clone = deferred.benchmark;
         }
-
         var bench = clone._original,
             fn = bench.fn,
             fnArg = deferred ? getFirstArgument(fn) || 'deferred' : '',
@@ -1588,6 +1587,45 @@
           'teardown': getSource(bench.teardown, interpolate('m#.teardown()'))
         });
 
+        // use API of chosen timer
+        if (timer.unit == 'ns') {
+          if (timer.ns.nanoTime) {
+            _.extend(templateData, {
+              'begin': interpolate('s#=n#.nanoTime()'),
+              'end': interpolate('r#=(n#.nanoTime()-s#)/1e9')
+            });
+          } else {
+            _.extend(templateData, {
+              'begin': interpolate('s#=n#()'),
+              'end': interpolate('r#=n#(s#);r#=r#[0]+(r#[1]/1e9)')
+            });
+          }
+        }
+        else if (timer.unit == 'us') {
+          if (timer.ns.stop) {
+            _.extend(templateData, {
+              'begin': interpolate('s#=n#.start()'),
+              'end': interpolate('r#=n#.microseconds()/1e6')
+            });
+          } else if (perfName) {
+            _.extend(templateData, {
+              'begin': interpolate('s#=n#.' + perfName + '()'),
+              'end': interpolate('r#=(n#.' + perfName + '()-s#)/1e3')
+            });
+          } else {
+            _.extend(templateData, {
+              'begin': interpolate('s#=n#()'),
+              'end': interpolate('r#=(n#()-s#)/1e6')
+            });
+          }
+        }
+        else {
+          _.extend(templateData, {
+            'begin': interpolate('s#=new n#'),
+            'end': interpolate('r#=(new n#-s#)/1e3')
+          });
+        }
+
         var count = bench.count = clone.count,
             decompilable = support.decompilation || stringable,
             id = bench.id,
@@ -1609,6 +1647,16 @@
             ns = timer.ns = new applet.Packages.nano;
           }
         }
+        // define `timer` methods
+        timer.start = createFunction(
+          interpolate('o#'),
+          interpolate('var n#=this.ns,${begin};o#.elapsed=0;o#.timeStamp=s#')
+        );
+
+        timer.stop = createFunction(
+          interpolate('o#'),
+          interpolate('var n#=this.ns,s#=o#.timeStamp,${end};o#.elapsed=r#')
+        );
 
         // Compile in setup/teardown functions and the test loop.
         // Create a new compiled test, instead of using the cached `bench.compiled`,
@@ -1643,7 +1691,7 @@
             // pretest to determine if compiled code is exits early, usually by a
             // rogue `return` statement, by checking for a return object with the uid
             bench.count = 1;
-            compiled = (compiled.call(bench, context, timer) || {}).uid == uid && compiled;
+            compiled = (compiled.call(bench, context, timer) || {}).uid == templateData.uid && compiled;
             bench.count = count;
           }
         } catch(e) {
@@ -1762,7 +1810,7 @@
        */
       function interpolate(string) {
         // replaces all occurrences of `#` with a unique number and template tokens with content
-        return _.template(string.replace(/\#/g, /\d+/.exec(uid)), templateData || {});
+        return _.template(string.replace(/\#/g, /\d+/.exec(templateData.uid)), templateData);
       }
 
       /*----------------------------------------------------------------------*/
@@ -1816,50 +1864,6 @@
       if (timer.res == Infinity) {
         throw new Error('Benchmark.js was unable to find a working timer.');
       }
-      // use API of chosen timer
-      if (timer.unit == 'ns') {
-        if (timer.ns.nanoTime) {
-          _.extend(templateData, {
-            'begin': interpolate('s#=n#.nanoTime()'),
-            'end': interpolate('r#=(n#.nanoTime()-s#)/1e9')
-          });
-        } else {
-          _.extend(templateData, {
-            'begin': interpolate('s#=n#()'),
-            'end': interpolate('r#=n#(s#);r#=r#[0]+(r#[1]/1e9)')
-          });
-        }
-      }
-      else if (timer.unit == 'us') {
-        if (timer.ns.stop) {
-          _.extend(templateData, {
-            'begin': interpolate('s#=n#.start()'),
-            'end': interpolate('r#=n#.microseconds()/1e6')
-          });
-        } else if (perfName) {
-          _.extend(templateData, {
-            'begin': interpolate('s#=n#.' + perfName + '()'),
-            'end': interpolate('r#=(n#.' + perfName + '()-s#)/1e3')
-          });
-        } else {
-          _.extend(templateData, {
-            'begin': interpolate('s#=n#()'),
-            'end': interpolate('r#=(n#()-s#)/1e6')
-          });
-        }
-      }
-
-      // define `timer` methods
-      timer.start = createFunction(
-        interpolate('o#'),
-        interpolate('var n#=this.ns,${begin};o#.elapsed=0;o#.timeStamp=s#')
-      );
-
-      timer.stop = createFunction(
-        interpolate('o#'),
-        interpolate('var n#=this.ns,s#=o#.timeStamp,${end};o#.elapsed=r#')
-      );
-
       // resolve time span required to achieve a percent uncertainty of at most 1%
       // http://spiff.rit.edu/classes/phys273/uncert/uncert.html
       options.minTime || (options.minTime = max(timer.res / 2 / 0.01, 0.05));
@@ -2356,6 +2360,11 @@
       'support': support
     });
 
+    // Add Lo-Dash methods to Benchmark
+    _.each(['each', 'forEach', 'forOwn', 'has', 'indexOf', 'map', 'pluck', 'reduce'], function(methodName) {
+      Benchmark[methodName] = _[methodName];
+    });
+
     /*------------------------------------------------------------------------*/
 
     _.extend(Benchmark.prototype, {
@@ -2777,11 +2786,12 @@
       'off': off,
       'on': on,
       'pop': arrayRef.pop,
-      'push': arrayRef.push,
+      'push': push,
       'reset': resetSuite,
       'run': runSuite,
       'reverse': arrayRef.reverse,
       'shift': shift,
+      'slice': arrayRef.slice,
       'sort': arrayRef.sort,
       'splice': arrayRef.splice,
       'unshift': arrayRef.unshift
@@ -2798,6 +2808,16 @@
 
     /*------------------------------------------------------------------------*/
 
+    // add Lo-Dash methods as Suite methods
+    _.each(['each', 'forEach', 'indexOf', 'map', 'pluck', 'reduce'], function(methodName) {
+      var func = _[methodName];
+      Suite.prototype[methodName] = function() {
+        var args = [this];
+        push.apply(args, arguments);
+        return func.apply(_, args);
+      };
+    });
+
     // avoid array-like object bugs with `Array#shift` and `Array#splice`
     // in Firefox < 10 and IE < 9
     if (!_.support.spliceObjects) {