Faster selector execution by pre-parsing fields

.gitignore

@@ -45,6 +45,7 @@ share/server/main-coffee.js
 share/server/main.js
 share/server/main-ast-bypass.js
 share/www
+src/argparse/
 src/bear/
 src/certifi/
 src/couch/priv/couch_js/**/config.h
@@ -54,6 +55,7 @@ src/couch/priv/couch_ejson_compare/couch_ejson_compare.d
 src/couch/priv/couch_js/**/*.d
 src/couch/priv/icu_driver/couch_icu_driver.d
 src/cowlib/
+src/erlperf/
 src/mango/src/mango_cursor_text.nocompile
 src/excoveralls/
 src/fauxton/

rebar.config.script

@@ -164,7 +164,9 @@ DepDescs = [
 {jiffy,            "jiffy",            {tag, "1.1.2"}},
 {mochiweb,         "mochiweb",         {tag, "v3.3.0"}},
 {meck,             "meck",             {tag, "v1.1.0"}},
-{recon,            "recon",            {tag, "2.5.6"}}
+{recon,            "recon",            {tag, "2.5.6"}},
+{argparse,         {url, "https://github.com/max-au/argparse"}, "1.2.4"},
+{erlperf,          {url, "https://github.com/max-au/erlperf"}, "2.3.0"}
 ].
 
 WithProper = lists:keyfind(with_proper, 1, CouchConfig) == {with_proper, true}.

src/mango/src/mango_cursor.erl

@@ -358,7 +358,7 @@ explain(#cursor{} = Cursor) ->
             {dbname, DbName},
             {index, JSON},
             {partitioned, Partitioned},
-            {selector, Selector},
+            {selector, mango_util:join_keys(Selector)},
             {opts, {Opts}},
             {limit, Limit},
             {skip, Skip},

src/mango/src/mango_cursor_view.erl

@@ -1016,7 +1016,7 @@ required_fields_disjoint_fields_test() ->
     ?assertEqual([<<"field1">>, <<"field2">>, <<"field3">>], required_fields(Cursor1)),
     Fields2 = [<<"field1">>, <<"field2">>],
     Selector2 = to_selector(#{<<"field3">> => undefined, <<"field4">> => undefined}),
-    Cursor2 = #cursor{fields = Fields2, selector = to_selector(Selector2)},
+    Cursor2 = #cursor{fields = Fields2, selector = Selector2},
     ?assertEqual(
         [<<"field1">>, <<"field2">>, <<"field3">>, <<"field4">>], required_fields(Cursor2)
     ).

src/mango/src/mango_idx_view.erl

@@ -233,7 +233,7 @@ opts() ->
 make_view(Idx) ->
     View =
         {[
-            {<<"map">>, Idx#idx.def},
+            {<<"map">>, mango_util:join_keys(Idx#idx.def)},
             {<<"reduce">>, <<"_count">>},
             {<<"options">>, {Idx#idx.opts}}
         ]},
@@ -271,23 +271,26 @@ validate_ddoc(VProps) ->
 % the equivalent of a multi-query. But that's for another
 % day.
 
+indexable_fields(Selector) ->
+    [mango_util:join_field(F) || F <- indexable_paths(Selector)].
+
 % We can see through '$and' trivially
-indexable_fields({[{<<"$and">>, Args}]}) ->
-    lists:usort(lists:flatten([indexable_fields(A) || A <- Args]));
+indexable_paths({[{<<"$and">>, Args}]}) ->
+    lists:usort(lists:flatmap(fun(A) -> indexable_paths(A) end, Args));
 % So far we can't see through any other operator
-indexable_fields({[{<<"$", _/binary>>, _}]}) ->
+indexable_paths({[{<<"$", _/binary>>, _}]}) ->
     [];
 % If we have a field with a terminator that is locatable
 % using an index then the field is a possible index
-indexable_fields({[{Field, Cond}]}) ->
+indexable_paths({[{Field, Cond}]}) ->
     case indexable(Cond) of
         true ->
             [Field];
         false ->
             []
     end;
 % An empty selector
-indexable_fields({[]}) ->
+indexable_paths({[]}) ->
     [].
 
 % Check if a condition is indexable. The logical
@@ -320,8 +323,8 @@ indexable({[{<<"$", _/binary>>, _}]}) ->
 
 % For each field, return {Field, Range}
 field_ranges(Selector) ->
-    Fields = indexable_fields(Selector),
-    field_ranges(Selector, Fields).
+    Fields = indexable_paths(Selector),
+    [{mango_util:join_field(F), R} || {F, R} <- field_ranges(Selector, Fields)].
 
 field_ranges(Selector, Fields) ->
     field_ranges(Selector, Fields, []).

src/mango/src/mango_selector.erl

@@ -38,7 +38,7 @@ normalize(Selector) ->
     ],
     {NProps} = lists:foldl(fun(Step, Sel) -> Step(Sel) end, Selector, Steps),
     FieldNames = [Name || {Name, _} <- NProps],
-    case lists:member(<<>>, FieldNames) of
+    case lists:member([], FieldNames) of
         true ->
             ?MANGO_ERROR({invalid_selector, missing_field_name});
         false ->
@@ -210,7 +210,7 @@ norm_ops(Value) ->
 norm_fields({[]}) ->
     {[]};
 norm_fields(Selector) ->
-    norm_fields(Selector, <<>>).
+    norm_fields(Selector, []).
 
 % Operators where we can push the field names further
 % down the operator tree
@@ -237,7 +237,7 @@ norm_fields({[{<<"$keyMapMatch">>, Arg}]}, Path) ->
 % $default field. This also asserts that the $default
 % field is at the root as well as that it only has
 % a $text operator applied.
-norm_fields({[{<<"$default">>, {[{<<"$text">>, _Arg}]}}]} = Sel, <<>>) ->
+norm_fields({[{<<"$default">>, {[{<<"$text">>, _Arg}]}}]} = Sel, []) ->
     Sel;
 norm_fields({[{<<"$default">>, _}]} = Selector, _) ->
     ?MANGO_ERROR({bad_field, Selector});
@@ -249,12 +249,11 @@ norm_fields({[{<<"$", _/binary>>, _}]} = Cond, Path) ->
 % We've found a field name. Append it to the path
 % and skip this node as we unroll the stack as
 % the full path will be further down the branch.
-norm_fields({[{Field, Cond}]}, <<>>) ->
-    % Don't include the '.' for the first element of
-    % the path.
-    norm_fields(Cond, Field);
-norm_fields({[{Field, Cond}]}, Path) ->
-    norm_fields(Cond, <<Path/binary, ".", Field/binary>>);
+norm_fields({[{Field, Cond}]}, Path) when is_binary(Field) ->
+    {ok, F} = mango_util:parse_field(Field),
+    norm_fields({[{F, Cond}]}, Path);
+norm_fields({[{Field, Cond}]}, Path) when is_list(Field) ->
+    norm_fields(Cond, Path ++ Field);
 % An empty selector
 norm_fields({[]}, Path) ->
     {Path, {[]}};
@@ -575,7 +574,9 @@ match({[_, _ | _] = _Props} = Sel, _Value, _Cmp) ->
 % match against.
 
 has_required_fields(Selector, RequiredFields) ->
-    Remainder = has_required_fields_int(Selector, RequiredFields),
+    Paths0 = [mango_util:parse_field(F) || F <- RequiredFields],
+    Paths = [P || {ok, P} <- Paths0],
+    Remainder = has_required_fields_int(Selector, Paths),
     Remainder == [].
 
 % Empty selector
@@ -634,6 +635,9 @@ has_required_fields_int([{[{Field, Cond}]} | Rest], RequiredFields) ->
     end.
 
 % Returns true if a field in the selector is a constant value e.g. {a: {$eq: 1}}
+is_constant_field(Selector, Field) when not is_list(Field) ->
+    {ok, Path} = mango_util:parse_field(Field),
+    is_constant_field(Selector, Path);
 is_constant_field({[]}, _Field) ->
     false;
 is_constant_field(Selector, Field) when not is_list(Selector) ->
@@ -653,7 +657,7 @@ is_constant_field([{[{_UnMatched, _}]} | Rest], Field) ->
 fields({[{<<"$", _/binary>>, Args}]}) when is_list(Args) ->
     lists:flatmap(fun fields/1, Args);
 fields({[{Field, _Cond}]}) ->
-    [Field];
+    [mango_util:join_field(Field)];
 fields({[]}) ->
     [].
 
@@ -1087,4 +1091,91 @@ match_beginswith_test() ->
         check_beginswith(<<"user_id">>, InvalidArg)
     ).
 
+bench(Cases) ->
+    Benches = lists:map(
+        fun({_, Sel0, Doc}) ->
+            Sel = normalize(Sel0),
+            #{runner => fun() -> match_int(Sel, Doc) end}
+        end,
+        Cases
+    ),
+    Results = erlperf:compare(Benches, #{}),
+    ?debugFmt("", []),
+    lists:foreach(
+        fun({{Name, _, _}, Result}) ->
+            ?debugFmt("bench [~s] = ~p", [Name, Result])
+        end,
+        lists:zip(Cases, Results)
+    ).
+
+bench_test() ->
+    bench([
+        {
+            "1 field",
+            {[{<<"a">>, 1}]},
+            {[{<<"a">>, 1}]}
+        },
+        {
+            "3 sibling fields",
+            {[{<<"a">>, 1}, {<<"b">>, 2}, {<<"c">>, 3}]},
+            {[{<<"a">>, 1}, {<<"b">>, 2}, {<<"c">>, 3}]}
+        },
+        {
+            "3 nested fields",
+            {[{<<"a">>, {[{<<"b">>, {[{<<"c">>, 1}]}}]}}]},
+            {[{<<"a">>, {[{<<"b">>, {[{<<"c">>, 1}]}}]}}]}
+        },
+        {
+            "allMatch: 1 field",
+            {[
+                {<<"a">>,
+                    {[
+                        {<<"$allMatch">>,
+                            {[
+                                {<<"b">>, {[{<<"$gt">>, 0}]}}
+                            ]}}
+                    ]}}
+            ]},
+            {[
+                {<<"a">>, [{[{<<"b">>, N}]} || N <- lists:seq(1, 10)]}
+            ]}
+        },
+        {
+            "allMatch: 3 sibling fields",
+            {[
+                {<<"a">>,
+                    {[
+                        {<<"$allMatch">>,
+                            {[
+                                {<<"b">>, {[{<<"$gt">>, 0}]}},
+                                {<<"c">>, {[{<<"$gt">>, 0}]}},
+                                {<<"d">>, {[{<<"$gt">>, 0}]}}
+                            ]}}
+                    ]}}
+            ]},
+            {[
+                {<<"a">>, [
+                    {[{<<"b">>, N}, {<<"c">>, N}, {<<"d">>, N}]}
+                 || N <- lists:seq(1, 10)
+                ]}
+            ]}
+        },
+        {
+            "allMatch: 3 nested fields",
+            {[
+                {<<"a">>,
+                    {[
+                        {<<"$allMatch">>,
+                            {[{<<"b">>, {[{<<"c">>, {[{<<"d">>, {[{<<"$gt">>, 0}]}}]}}]}}]}}
+                    ]}}
+            ]},
+            {[
+                {<<"a">>, [
+                    {[{<<"b">>, {[{<<"c">>, {[{<<"d">>, N}]}}]}}]}
+                 || N <- lists:seq(1, 10)
+                ]}
+            ]}
+        }
+    ]).
+
 -endif.

src/mango/src/mango_selector_text.erl

@@ -177,10 +177,10 @@ convert(_Path, {[{<<"$", _/binary>> = Op, _}]}) ->
 convert(Path, {[{Field0, Cond}]}) ->
     {ok, PP0} =
         case Field0 of
-            <<>> ->
-                {ok, []};
-            _ ->
-                mango_util:parse_field(Field0)
+            F when is_binary(F) ->
+                mango_util:parse_field(F);
+            F when is_list(F) ->
+                {ok, F}
         end,
     % Later on, we perform a lucene_escape_user call on the
     % final Path, which calls parse_field again. Calling the function

src/mango/src/mango_util.erl

@@ -41,6 +41,8 @@
     join/2,
 
     parse_field/1,
+    join_field/1,
+    join_keys/1,
 
     cached_re/2
 ]).
@@ -370,6 +372,25 @@ parse_field_slow(Field) ->
     ),
     {ok, Path}.
 
+join_keys({Sel}) when is_list(Sel) ->
+    Pairs = [{join_field(K), join_keys(V)} || {K, V} <- Sel],
+    {Pairs};
+join_keys(Sel) when is_list(Sel) ->
+    [join_keys(S) || S <- Sel];
+join_keys(Sel) ->
+    Sel.
+
+join_field(Field) when is_list(Field) ->
+    Parts = [field_to_binary(F) || F <- Field],
+    binary:join(Parts, <<".">>);
+join_field(Field) ->
+    Field.
+
+field_to_binary(Field) when is_list(Field) ->
+    list_to_binary(Field);
+field_to_binary(Field) when is_binary(Field) ->
+    Field.
+
 check_non_empty(Field, Parts) ->
     case lists:member(<<>>, Parts) of
         true ->

test/fixtures/allowed-xref.txt

@@ -1,2 +1,6 @@
+src/erlperf_cli.erl:{102,1}: Warning: erlperf_cli:main/1 calls undefined function args:format_error/3 (Xref)
+src/erlperf_cli.erl:{102,1}: Warning: erlperf_cli:main/1 calls undefined function args:parse/3 (Xref)
 src/ioq.erl: Warning: ioq:get_disk_queues/0 is undefined function (Xref)
 src/weatherreport_check_ioq.erl:{95,1}: Warning: weatherreport_check_ioq:check_legacy_int/1 calls undefined function ioq:get_disk_queues/0 (Xref)
+Warning: args:format_error/3 is undefined function (Xref)
+Warning: args:parse/3 is undefined function (Xref)