speed up by using acorn tokenizer when possible

find-fast.js

@@ -0,0 +1,113 @@
+var acorn = require('acorn-node');
+var defined = require('defined');
+
+var ST_NONE = 0; // Default.
+var ST_SAW_NAME = 1; // Saw a `require` identifier.
+var ST_INSIDE_CALL = 2; // Found a `require(` sequence; if followed by a string, that is a dependency.
+var ST_MEMBER_EXPRESSION = 3; // Saw a `.`; if followed by a `require` identifier that should be ignored.
+var ST_REDEF_PATTERN = 4; // Currently in progress detecting a redefinition pattern: `{0:[function(require`
+var ST_REDEFINED = 5; // Currently inside a scope with a redefined `require` identifier.
+
+var REQUIRE_REDEF_PATTERN = [
+    function (token) { return token.type === acorn.tokTypes.braceL; }, // {
+    function (token) { return token.type === acorn.tokTypes.num || token.type === acorn.tokTypes.string; }, // 0
+    function (token) { return token.type === acorn.tokTypes.colon; }, // :
+    function (token) { return token.type === acorn.tokTypes.bracketL; }, // [
+    function (token) { return token.type === acorn.tokTypes._function; }, // function
+    function (token) { return token.type === acorn.tokTypes.parenL; }, // (
+    function (token, opts) { return token.type === acorn.tokTypes.name && token.value === opts.word; }, // require
+];
+
+module.exports = function findFast(src, opts) {
+    if (!opts) opts = {};
+    if (typeof src !== 'string') src = String(src);
+    if (opts.word === undefined) opts.word = 'require';
+
+    var tokenizer = acorn.tokenizer(src, opts.parse);
+    var token;
+    var state = ST_NONE;
+    // Current index in the require redefinition pattern.
+    var redefIndex = 0;
+    // Block scope depth when require was redefined. This is used to match the
+    // correct } with the opening { after the redefining function parameter list.
+    var redefDepth = 0;
+
+    var opener;
+    var args = [];
+
+    var modules = { strings: [], expressions: [] };
+    if (opts.nodes) modules.nodes = [];
+
+    while ((token = tokenizer.getToken()) && token.type !== acorn.tokTypes.eof) {
+        if (state === ST_REDEFINED) {
+            if (token.type === acorn.tokTypes.braceL) redefDepth++;
+            if (token.type === acorn.tokTypes.braceR) redefDepth--;
+            if (redefDepth === 0) {
+                state = ST_NONE;
+            }
+            continue;
+        }
+        if (state === ST_REDEF_PATTERN) {
+            if (redefIndex >= REQUIRE_REDEF_PATTERN.length) {
+                // the { after the function() parameter list
+                if (token.type === acorn.tokTypes.braceL) {
+                    state = ST_REDEFINED;
+                    redefDepth = 1;
+                }
+                continue;
+            } else if (REQUIRE_REDEF_PATTERN[redefIndex](token, opts)) {
+                redefIndex++;
+                continue;
+            } else {
+                redefIndex = 0;
+                state = ST_NONE;
+            }
+        }
+
+        if (state !== ST_INSIDE_CALL && token.type === acorn.tokTypes.dot) {
+            state = ST_MEMBER_EXPRESSION;
+        } else if (state === ST_NONE && token.type === acorn.tokTypes.name && mayBeRequire(token)) {
+            state = ST_SAW_NAME;
+            opener = token;
+        } else if (state === ST_SAW_NAME && token.type === acorn.tokTypes.parenL) {
+            state = ST_INSIDE_CALL;
+            args = [];
+        } else if (state === ST_INSIDE_CALL) {
+            if (token.type === acorn.tokTypes.parenR) { // End of fn() call
+                if (args.length === 1 && args[0].type === acorn.tokTypes.string) {
+                    modules.strings.push(args[0].value);
+                } else if (args.length === 3 // A template string without any expressions
+                          && args[0].type === acorn.tokTypes.backQuote
+                          && args[1].type === acorn.tokTypes.template
+                          && args[2].type === acorn.tokTypes.backQuote) {
+                    modules.strings.push(args[1].value);
+                } else if (args.length > 0) {
+                    modules.expressions.push(src.slice(args[0].start, args[args.length - 1].end));
+                }
+
+                if (opts.nodes) {
+                    // Cut `src` at the end of this call, so that parseExpressionAt doesn't consider the `.abc` in
+                    // `require('xyz').abc`
+                    var chunk = src.slice(0, token.end);
+                    var node = acorn.parseExpressionAt(chunk, opener.start, opts.parse);
+                    modules.nodes.push(node);
+                }
+
+                state = ST_NONE;
+            } else {
+                args.push(token);
+            }
+        } else if (REQUIRE_REDEF_PATTERN[0](token)) {
+            state = ST_REDEF_PATTERN;
+            redefIndex = 1;
+        } else {
+            state = ST_NONE;
+        }
+    }
+    return modules;
+
+    function mayBeRequire(token) {
+        return token.type === acorn.tokTypes.name &&
+            token.value === opts.word;
+    }
+}

index.js

@@ -1,12 +1,14 @@
 var acorn = require('acorn-node');
 var walk = require('acorn-node/walk');
+var copy = require('shallow-copy');
 var defined = require('defined');
+var fastFind = require('./find-fast');
 
 var requireRe = /\brequire\b/;
 
-function parse (src, opts) {
-    if (!opts) opts = {};
-    return acorn.parse(src, {
+function getParseOpts (opts) {
+    opts = opts || {};
+    return {
         ecmaVersion: defined(opts.ecmaVersion, 9),
         sourceType: defined(opts.sourceType, 'script'),
         ranges: defined(opts.ranges, opts.range),
@@ -19,7 +21,7 @@ function parse (src, opts) {
             opts.allowImportExportEverywhere, true
         ),
         allowHashBang: defined(opts.allowHashBang, true)
-    });
+    };
 }
 
 var exports = module.exports = function (src, opts) {
@@ -28,6 +30,12 @@ var exports = module.exports = function (src, opts) {
 
 exports.find = function (src, opts) {
     if (!opts) opts = {};
+    else opts = copy(opts);
+    opts.parse = getParseOpts(opts.parse);
+
+    if (!opts.isRequire && !opts.fullParse) {
+        return fastFind(src, opts);
+    }
 
     var word = opts.word === undefined ? 'require' : opts.word;
     if (typeof src !== 'string') src = String(src);
@@ -44,11 +52,12 @@ exports.find = function (src, opts) {
     var wordRe = word === 'require' ? requireRe : RegExp('\\b' + word + '\\b');
     if (!wordRe.test(src)) return modules;
 
-    var ast = parse(src, opts.parse);
+    var ast = acorn.parse(src, opts.parse);
 
     function visit(node, st, c) {
         var hasRequire = wordRe.test(src.slice(node.start, node.end));
         if (!hasRequire) return;
+        if (isBundledDefinition(node)) return;
         walk.base[node.type](node, st, c);
         if (node.type !== 'CallExpression') return;
         if (isRequire(node)) {
@@ -75,6 +84,20 @@ exports.find = function (src, opts) {
         Statement: visit,
         Expression: visit
     });
+
+    // Detect `require` redefinitions in function parameter lists, like
+    // in `{0:[function(require,module,exports){` generated by browser-pack.
+    // This is a simple way to address the 99% case without doing full scope analysis
+    function isBundledDefinition(node) {
+        if (node.type !== 'ObjectExpression') return false;
+        if (node.properties.length < 1) return false;
+        var arr = node.properties[0].value;
+        if (arr.type !== 'ArrayExpression') return false;
+        if (arr.elements.length < 2) return false;
+        if (arr.elements[0].type !== 'FunctionExpression') return false;
+        var fn = arr.elements[0];
+        return fn.params.length > 0 && fn.params[0].type === 'Identifier' && fn.params[0].name === word;
+    }
 
     return modules;
 };

package.json

@@ -11,7 +11,8 @@
   "dependencies": {
     "acorn-node": "^1.3.0",
     "defined": "^1.0.0",
-    "minimist": "^1.1.1"
+    "minimist": "^1.1.1",
+    "shallow-copy": "0.0.1"
   },
   "devDependencies": {
     "tap": "^10.7.3"

test/both.js

@@ -11,6 +11,14 @@ test('both', function (t) {
     t.end();
 });
 
+test('both fullParse', function (t) {
+    var modules = detective.find(src, { fullParse: true });
+    t.deepEqual(modules.strings, [ 'a', 'b' ]);
+    t.deepEqual(modules.expressions, [ "'c' + x", "'d' + y" ]);
+    t.notOk(modules.nodes, 'has no nodes');
+    t.end();
+});
+
 test('both with nodes specified in opts', function (t) {
     var modules = detective.find(src, { nodes: true });
     t.deepEqual(modules.strings, [ 'a', 'b' ]);
@@ -24,3 +32,17 @@ test('both with nodes specified in opts', function (t) {
       'has a node for each require');
     t.end();
 });
+
+test('both with nodes and fullParse', function (t) {
+    var modules = detective.find(src, { nodes: true, fullParse: true });
+    t.deepEqual(modules.strings, [ 'a', 'b' ]);
+    t.deepEqual(modules.expressions, [ "'c' + x", "'d' + y" ]);
+    t.deepEqual(
+      modules.nodes.map(function (n) {
+        var arg = n.arguments[0];
+        return arg.value || arg.left.value;
+      }),
+      [ 'a', 'b', 'c', 'd' ],
+      'has a node for each require');
+    t.end();
+});

test/chained.js

@@ -5,5 +5,6 @@ var src = fs.readFileSync(__dirname + '/files/chained.js');
 
 test('chained', function (t) {
     t.deepEqual(detective(src), [ 'c', 'b', 'a' ]);
+    t.deepEqual(detective(src, { fullParse: true }), [ 'c', 'b', 'a' ]);
     t.end();
 });

test/comment.js

@@ -0,0 +1,18 @@
+var test = require('tap').test;
+var detective = require('../');
+var fs = require('fs');
+var src = fs.readFileSync(__dirname + '/files/comment.js');
+
+test('comment', function (t) {
+    var modules = detective.find(src);
+    t.deepEqual(modules.strings, [ 'beep' ]);
+    t.notOk(modules.nodes, 'has no nodes');
+    t.end();
+});
+
+test('comment fullParse', function (t) {
+    var modules = detective.find(src, { fullParse: true });
+    t.deepEqual(modules.strings, [ 'beep' ]);
+    t.notOk(modules.nodes, 'has no nodes');
+    t.end();
+});

test/complicated.js

@@ -51,8 +51,9 @@ var sources = [
 ];
 
 test('complicated', function (t) {
-    t.plan(sources.length);
+    t.plan(sources.length * 2);
     sources.forEach(function(src) {
         t.deepEqual(detective(src), [ 'a' ]);
+        t.deepEqual(detective(src, { fullParse: true }), [ 'a' ]);
     });
 });

test/es6-module.js

@@ -4,6 +4,7 @@ var fs = require('fs');
 var src = fs.readFileSync(__dirname + '/files/es6-module.js');
 
 test('es6-module', function (t) {
-    t.plan(1);
+    t.plan(2);
     t.deepEqual(detective(src, {parse: {sourceType: 'module'}}), [ 'a', 'b' ]);
+    t.deepEqual(detective(src, {parse: {sourceType: 'module'}, fullParse: true}), [ 'a', 'b' ]);
 });

test/files/comment.js

@@ -0,0 +1,5 @@
+var x = require /* idk */
+// whatever
+(
+'beep' // boop
+)

test/files/scope.js

@@ -0,0 +1,12 @@
+(function(modules){
+    modules[1](function(i){return modules[i]()})
+})({1:[function (require,module,exports) {
+    require('./y') // inside a bundle; should not be detected
+},{'./y':2}],2:function(require,module,exports){
+    console.log("abc")
+}})
+
+(function (require) {
+    require('./x'); // not inside a bundle; should be detected
+}(require)); // (because someone might do this)
+require('./z')

test/generators.js

@@ -4,6 +4,7 @@ var fs = require('fs');
 var src = fs.readFileSync(__dirname + '/files/generators.js');
 
 test('generators', function (t) {
-    t.plan(1);
+    t.plan(2);
     t.deepEqual(detective(src), [ 'a', 'b' ]);
+    t.deepEqual(detective(src, { fullParse: true }), [ 'a', 'b' ]);
 });

test/nested.js

@@ -5,5 +5,6 @@ var src = fs.readFileSync(__dirname + '/files/nested.js');
 
 test('nested', function (t) {
     t.deepEqual(detective(src), [ 'a', 'b', 'c' ]);
+    t.deepEqual(detective(src, { fullParse: true }), [ 'a', 'b', 'c' ]);
     t.end();
 });

test/noargs.js

@@ -7,8 +7,9 @@ var fs = require('fs');
 var src = [ 'fn();', 'otherfn();', 'fn();' ].join('\n')
 
 test('noargs', function (t) {
-    t.plan(1);
+    t.plan(2);
     t.deepEqual(detective(src, { word: 'fn' }).length, 0, 'finds no arg id');
+    t.deepEqual(detective(src, { word: 'fn', fullParse: true }).length, 0, 'finds no arg id');
 });
 
 test('find noargs with nodes', function (t) {
@@ -24,3 +25,17 @@ test('find noargs with nodes', function (t) {
       'all matches are correct'
     );
 });
+
+test('find noargs with nodes and fullParse', function (t) {
+    t.plan(4);
+    var modules = detective.find(src, { word: 'fn', nodes: true, fullParse: true });
+    t.equal(modules.strings.length, 0, 'finds no arg id');
+    t.equal(modules.expressions.length, 0, 'finds no expressions');
+    t.equal(modules.nodes.length, 2, 'finds a node for each matching function call');
+    t.equal(
+      modules.nodes.filter(function (x) { 
+        return x.callee.name === 'fn' 
+      }).length, 2, 
+      'all matches are correct'
+    );
+});

test/return.js

@@ -4,6 +4,7 @@ var fs = require('fs');
 var src = [ 'require("a")\nreturn' ];
 
 test('return', function (t) {
-    t.plan(1);
+    t.plan(2);
     t.deepEqual(detective(src), [ 'a' ]);
+    t.deepEqual(detective(src, { fullParse: true }), [ 'a' ]);
 });

test/scope.js

@@ -0,0 +1,10 @@
+var test = require('tap').test;
+var detective = require('../');
+var fs = require('fs');
+var src = fs.readFileSync(__dirname + '/files/scope.js');
+
+test('scope', function (t) {
+    t.plan(2);
+    t.deepEqual(detective(src), [ './x', './z' ]);
+    t.deepEqual(detective(src, { fullParse: true }), [ './x', './z' ]);
+});

test/set-in-object-pattern.js

@@ -8,5 +8,9 @@ test('set in object pattern', function (t) {
         detective(src, { word : 'load' }),
         [ 'a', 'b', 'c', 'tt' ]
     );
+    t.deepEqual(
+        detective(src, { word : 'load', fullParse: true }),
+        [ 'a', 'b', 'c', 'tt' ]
+    );
     t.end();
-});
+});

test/shebang.js

@@ -4,6 +4,7 @@ var fs = require('fs');
 var src = fs.readFileSync(__dirname + '/files/shebang.js');
 
 test('shebang', function (t) {
-    t.plan(1);
+    t.plan(2);
     t.deepEqual(detective(src), [ 'a', 'b', 'c' ]);
+    t.deepEqual(detective(src, { fullParse: true }), [ 'a', 'b', 'c' ]);
 });

test/sparse-array.js

@@ -8,6 +8,9 @@ test('sparse-array', function (t) {
     t.doesNotThrow(function () {
       detective(src)
     })
+    t.doesNotThrow(function () {
+      detective(src, { fullParse: true })
+    })
     t.end();
 });
 

test/strings.js

@@ -5,5 +5,6 @@ var src = fs.readFileSync(__dirname + '/files/strings.js');
 
 test('single', function (t) {
     t.deepEqual(detective(src), [ 'a', 'b', 'c', 'events', 'doom', 'y', 'events2' ]);
+    t.deepEqual(detective(src, { fullParse: true }), [ 'a', 'b', 'c', 'events', 'doom', 'y', 'events2' ]);
     t.end();
 });