diff --git a/.gitignore b/.gitignore index d46777512b7..b9e3ec42541 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ share/server/main-coffee.js share/server/main.js share/server/main-ast-bypass.js share/www +src/argparse/ src/bear/ src/certifi/ src/couch/priv/couch_js/**/config.h @@ -54,6 +55,7 @@ src/couch/priv/couch_ejson_compare/couch_ejson_compare.d src/couch/priv/couch_js/**/*.d src/couch/priv/icu_driver/couch_icu_driver.d src/cowlib/ +src/erlperf/ src/mango/src/mango_cursor_text.nocompile src/excoveralls/ src/fauxton/ diff --git a/rebar.config.script b/rebar.config.script index efc03a35e64..91c91f4a1dd 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -164,7 +164,9 @@ DepDescs = [ {jiffy, "jiffy", {tag, "1.1.2"}}, {mochiweb, "mochiweb", {tag, "v3.3.0"}}, {meck, "meck", {tag, "v1.1.0"}}, -{recon, "recon", {tag, "2.5.6"}} +{recon, "recon", {tag, "2.5.6"}}, +{argparse, {url, "https://github.com/max-au/argparse"}, "1.2.4"}, +{erlperf, {url, "https://github.com/max-au/erlperf"}, "2.3.0"} ]. WithProper = lists:keyfind(with_proper, 1, CouchConfig) == {with_proper, true}. diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl index b78f3682e79..2535a3bce4d 100644 --- a/src/mango/src/mango_cursor.erl +++ b/src/mango/src/mango_cursor.erl @@ -358,7 +358,7 @@ explain(#cursor{} = Cursor) -> {dbname, DbName}, {index, JSON}, {partitioned, Partitioned}, - {selector, Selector}, + {selector, mango_util:join_keys(Selector)}, {opts, {Opts}}, {limit, Limit}, {skip, Skip}, diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl index 0928ae19311..b496961f6dc 100644 --- a/src/mango/src/mango_cursor_view.erl +++ b/src/mango/src/mango_cursor_view.erl @@ -1016,7 +1016,7 @@ required_fields_disjoint_fields_test() -> ?assertEqual([<<"field1">>, <<"field2">>, <<"field3">>], required_fields(Cursor1)), Fields2 = [<<"field1">>, <<"field2">>], Selector2 = to_selector(#{<<"field3">> => undefined, <<"field4">> => undefined}), - Cursor2 = #cursor{fields = Fields2, selector = to_selector(Selector2)}, + Cursor2 = #cursor{fields = Fields2, selector = Selector2}, ?assertEqual( [<<"field1">>, <<"field2">>, <<"field3">>, <<"field4">>], required_fields(Cursor2) ). diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl index e2d29200b2e..d85efe81faf 100644 --- a/src/mango/src/mango_idx_view.erl +++ b/src/mango/src/mango_idx_view.erl @@ -233,7 +233,7 @@ opts() -> make_view(Idx) -> View = {[ - {<<"map">>, Idx#idx.def}, + {<<"map">>, mango_util:join_keys(Idx#idx.def)}, {<<"reduce">>, <<"_count">>}, {<<"options">>, {Idx#idx.opts}} ]}, @@ -271,15 +271,18 @@ validate_ddoc(VProps) -> % the equivalent of a multi-query. But that's for another % day. +indexable_fields(Selector) -> + [mango_util:join_field(F) || F <- indexable_paths(Selector)]. + % We can see through '$and' trivially -indexable_fields({[{<<"$and">>, Args}]}) -> - lists:usort(lists:flatten([indexable_fields(A) || A <- Args])); +indexable_paths({[{<<"$and">>, Args}]}) -> + lists:usort(lists:flatmap(fun(A) -> indexable_paths(A) end, Args)); % So far we can't see through any other operator -indexable_fields({[{<<"$", _/binary>>, _}]}) -> +indexable_paths({[{<<"$", _/binary>>, _}]}) -> []; % If we have a field with a terminator that is locatable % using an index then the field is a possible index -indexable_fields({[{Field, Cond}]}) -> +indexable_paths({[{Field, Cond}]}) -> case indexable(Cond) of true -> [Field]; @@ -287,7 +290,7 @@ indexable_fields({[{Field, Cond}]}) -> [] end; % An empty selector -indexable_fields({[]}) -> +indexable_paths({[]}) -> []. % Check if a condition is indexable. The logical @@ -320,8 +323,8 @@ indexable({[{<<"$", _/binary>>, _}]}) -> % For each field, return {Field, Range} field_ranges(Selector) -> - Fields = indexable_fields(Selector), - field_ranges(Selector, Fields). + Fields = indexable_paths(Selector), + [{mango_util:join_field(F), R} || {F, R} <- field_ranges(Selector, Fields)]. field_ranges(Selector, Fields) -> field_ranges(Selector, Fields, []). diff --git a/src/mango/src/mango_selector.erl b/src/mango/src/mango_selector.erl index 9c5b7a96f7f..578ea69764f 100644 --- a/src/mango/src/mango_selector.erl +++ b/src/mango/src/mango_selector.erl @@ -38,7 +38,7 @@ normalize(Selector) -> ], {NProps} = lists:foldl(fun(Step, Sel) -> Step(Sel) end, Selector, Steps), FieldNames = [Name || {Name, _} <- NProps], - case lists:member(<<>>, FieldNames) of + case lists:member([], FieldNames) of true -> ?MANGO_ERROR({invalid_selector, missing_field_name}); false -> @@ -210,7 +210,7 @@ norm_ops(Value) -> norm_fields({[]}) -> {[]}; norm_fields(Selector) -> - norm_fields(Selector, <<>>). + norm_fields(Selector, []). % Operators where we can push the field names further % down the operator tree @@ -237,7 +237,7 @@ norm_fields({[{<<"$keyMapMatch">>, Arg}]}, Path) -> % $default field. This also asserts that the $default % field is at the root as well as that it only has % a $text operator applied. -norm_fields({[{<<"$default">>, {[{<<"$text">>, _Arg}]}}]} = Sel, <<>>) -> +norm_fields({[{<<"$default">>, {[{<<"$text">>, _Arg}]}}]} = Sel, []) -> Sel; norm_fields({[{<<"$default">>, _}]} = Selector, _) -> ?MANGO_ERROR({bad_field, Selector}); @@ -249,12 +249,11 @@ norm_fields({[{<<"$", _/binary>>, _}]} = Cond, Path) -> % We've found a field name. Append it to the path % and skip this node as we unroll the stack as % the full path will be further down the branch. -norm_fields({[{Field, Cond}]}, <<>>) -> - % Don't include the '.' for the first element of - % the path. - norm_fields(Cond, Field); -norm_fields({[{Field, Cond}]}, Path) -> - norm_fields(Cond, <>); +norm_fields({[{Field, Cond}]}, Path) when is_binary(Field) -> + {ok, F} = mango_util:parse_field(Field), + norm_fields({[{F, Cond}]}, Path); +norm_fields({[{Field, Cond}]}, Path) when is_list(Field) -> + norm_fields(Cond, Path ++ Field); % An empty selector norm_fields({[]}, Path) -> {Path, {[]}}; @@ -575,7 +574,9 @@ match({[_, _ | _] = _Props} = Sel, _Value, _Cmp) -> % match against. has_required_fields(Selector, RequiredFields) -> - Remainder = has_required_fields_int(Selector, RequiredFields), + Paths0 = [mango_util:parse_field(F) || F <- RequiredFields], + Paths = [P || {ok, P} <- Paths0], + Remainder = has_required_fields_int(Selector, Paths), Remainder == []. % Empty selector @@ -634,6 +635,9 @@ has_required_fields_int([{[{Field, Cond}]} | Rest], RequiredFields) -> end. % Returns true if a field in the selector is a constant value e.g. {a: {$eq: 1}} +is_constant_field(Selector, Field) when not is_list(Field) -> + {ok, Path} = mango_util:parse_field(Field), + is_constant_field(Selector, Path); is_constant_field({[]}, _Field) -> false; is_constant_field(Selector, Field) when not is_list(Selector) -> @@ -653,7 +657,7 @@ is_constant_field([{[{_UnMatched, _}]} | Rest], Field) -> fields({[{<<"$", _/binary>>, Args}]}) when is_list(Args) -> lists:flatmap(fun fields/1, Args); fields({[{Field, _Cond}]}) -> - [Field]; + [mango_util:join_field(Field)]; fields({[]}) -> []. @@ -1087,4 +1091,91 @@ match_beginswith_test() -> check_beginswith(<<"user_id">>, InvalidArg) ). +bench(Cases) -> + Benches = lists:map( + fun({_, Sel0, Doc}) -> + Sel = normalize(Sel0), + #{runner => fun() -> match_int(Sel, Doc) end} + end, + Cases + ), + Results = erlperf:compare(Benches, #{}), + ?debugFmt("", []), + lists:foreach( + fun({{Name, _, _}, Result}) -> + ?debugFmt("bench [~s] = ~p", [Name, Result]) + end, + lists:zip(Cases, Results) + ). + +bench_test() -> + bench([ + { + "1 field", + {[{<<"a">>, 1}]}, + {[{<<"a">>, 1}]} + }, + { + "3 sibling fields", + {[{<<"a">>, 1}, {<<"b">>, 2}, {<<"c">>, 3}]}, + {[{<<"a">>, 1}, {<<"b">>, 2}, {<<"c">>, 3}]} + }, + { + "3 nested fields", + {[{<<"a">>, {[{<<"b">>, {[{<<"c">>, 1}]}}]}}]}, + {[{<<"a">>, {[{<<"b">>, {[{<<"c">>, 1}]}}]}}]} + }, + { + "allMatch: 1 field", + {[ + {<<"a">>, + {[ + {<<"$allMatch">>, + {[ + {<<"b">>, {[{<<"$gt">>, 0}]}} + ]}} + ]}} + ]}, + {[ + {<<"a">>, [{[{<<"b">>, N}]} || N <- lists:seq(1, 10)]} + ]} + }, + { + "allMatch: 3 sibling fields", + {[ + {<<"a">>, + {[ + {<<"$allMatch">>, + {[ + {<<"b">>, {[{<<"$gt">>, 0}]}}, + {<<"c">>, {[{<<"$gt">>, 0}]}}, + {<<"d">>, {[{<<"$gt">>, 0}]}} + ]}} + ]}} + ]}, + {[ + {<<"a">>, [ + {[{<<"b">>, N}, {<<"c">>, N}, {<<"d">>, N}]} + || N <- lists:seq(1, 10) + ]} + ]} + }, + { + "allMatch: 3 nested fields", + {[ + {<<"a">>, + {[ + {<<"$allMatch">>, + {[{<<"b">>, {[{<<"c">>, {[{<<"d">>, {[{<<"$gt">>, 0}]}}]}}]}}]}} + ]}} + ]}, + {[ + {<<"a">>, [ + {[{<<"b">>, {[{<<"c">>, {[{<<"d">>, N}]}}]}}]} + || N <- lists:seq(1, 10) + ]} + ]} + } + ]). + -endif. diff --git a/src/mango/src/mango_selector_text.erl b/src/mango/src/mango_selector_text.erl index fc30a57b614..8b5152dd151 100644 --- a/src/mango/src/mango_selector_text.erl +++ b/src/mango/src/mango_selector_text.erl @@ -177,10 +177,10 @@ convert(_Path, {[{<<"$", _/binary>> = Op, _}]}) -> convert(Path, {[{Field0, Cond}]}) -> {ok, PP0} = case Field0 of - <<>> -> - {ok, []}; - _ -> - mango_util:parse_field(Field0) + F when is_binary(F) -> + mango_util:parse_field(F); + F when is_list(F) -> + {ok, F} end, % Later on, we perform a lucene_escape_user call on the % final Path, which calls parse_field again. Calling the function diff --git a/src/mango/src/mango_util.erl b/src/mango/src/mango_util.erl index 837cbf3dbe8..58fd2aadf87 100644 --- a/src/mango/src/mango_util.erl +++ b/src/mango/src/mango_util.erl @@ -41,6 +41,8 @@ join/2, parse_field/1, + join_field/1, + join_keys/1, cached_re/2 ]). @@ -370,6 +372,25 @@ parse_field_slow(Field) -> ), {ok, Path}. +join_keys({Sel}) when is_list(Sel) -> + Pairs = [{join_field(K), join_keys(V)} || {K, V} <- Sel], + {Pairs}; +join_keys(Sel) when is_list(Sel) -> + [join_keys(S) || S <- Sel]; +join_keys(Sel) -> + Sel. + +join_field(Field) when is_list(Field) -> + Parts = [field_to_binary(F) || F <- Field], + binary:join(Parts, <<".">>); +join_field(Field) -> + Field. + +field_to_binary(Field) when is_list(Field) -> + list_to_binary(Field); +field_to_binary(Field) when is_binary(Field) -> + Field. + check_non_empty(Field, Parts) -> case lists:member(<<>>, Parts) of true -> diff --git a/test/fixtures/allowed-xref.txt b/test/fixtures/allowed-xref.txt index c630fc109e1..8292d692b0f 100644 --- a/test/fixtures/allowed-xref.txt +++ b/test/fixtures/allowed-xref.txt @@ -1,2 +1,6 @@ +src/erlperf_cli.erl:{102,1}: Warning: erlperf_cli:main/1 calls undefined function args:format_error/3 (Xref) +src/erlperf_cli.erl:{102,1}: Warning: erlperf_cli:main/1 calls undefined function args:parse/3 (Xref) src/ioq.erl: Warning: ioq:get_disk_queues/0 is undefined function (Xref) src/weatherreport_check_ioq.erl:{95,1}: Warning: weatherreport_check_ioq:check_legacy_int/1 calls undefined function ioq:get_disk_queues/0 (Xref) +Warning: args:format_error/3 is undefined function (Xref) +Warning: args:parse/3 is undefined function (Xref)