From 6f2123dc22da7ce16e4f74e0196b67f9ec52062b Mon Sep 17 00:00:00 2001 From: James Coglan Date: Mon, 26 Jan 2026 11:10:58 +0000 Subject: [PATCH 1/4] [wip] install erlperf --- .gitignore | 2 ++ rebar.config.script | 4 +++- test/fixtures/allowed-xref.txt | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d46777512b7..b9e3ec42541 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ share/server/main-coffee.js share/server/main.js share/server/main-ast-bypass.js share/www +src/argparse/ src/bear/ src/certifi/ src/couch/priv/couch_js/**/config.h @@ -54,6 +55,7 @@ src/couch/priv/couch_ejson_compare/couch_ejson_compare.d src/couch/priv/couch_js/**/*.d src/couch/priv/icu_driver/couch_icu_driver.d src/cowlib/ +src/erlperf/ src/mango/src/mango_cursor_text.nocompile src/excoveralls/ src/fauxton/ diff --git a/rebar.config.script b/rebar.config.script index efc03a35e64..91c91f4a1dd 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -164,7 +164,9 @@ DepDescs = [ {jiffy, "jiffy", {tag, "1.1.2"}}, {mochiweb, "mochiweb", {tag, "v3.3.0"}}, {meck, "meck", {tag, "v1.1.0"}}, -{recon, "recon", {tag, "2.5.6"}} +{recon, "recon", {tag, "2.5.6"}}, +{argparse, {url, "https://github.com/max-au/argparse"}, "1.2.4"}, +{erlperf, {url, "https://github.com/max-au/erlperf"}, "2.3.0"} ]. WithProper = lists:keyfind(with_proper, 1, CouchConfig) == {with_proper, true}. diff --git a/test/fixtures/allowed-xref.txt b/test/fixtures/allowed-xref.txt index c630fc109e1..8292d692b0f 100644 --- a/test/fixtures/allowed-xref.txt +++ b/test/fixtures/allowed-xref.txt @@ -1,2 +1,6 @@ +src/erlperf_cli.erl:{102,1}: Warning: erlperf_cli:main/1 calls undefined function args:format_error/3 (Xref) +src/erlperf_cli.erl:{102,1}: Warning: erlperf_cli:main/1 calls undefined function args:parse/3 (Xref) src/ioq.erl: Warning: ioq:get_disk_queues/0 is undefined function (Xref) src/weatherreport_check_ioq.erl:{95,1}: Warning: weatherreport_check_ioq:check_legacy_int/1 calls undefined function ioq:get_disk_queues/0 (Xref) +Warning: args:format_error/3 is undefined function (Xref) +Warning: args:parse/3 is undefined function (Xref) From bc3956eeea74aaa591a7b2cbb41101ea454b8b82 Mon Sep 17 00:00:00 2001 From: James Coglan Date: Wed, 28 Jan 2026 11:52:53 +0000 Subject: [PATCH 2/4] [wip] benchmarks --- src/mango/src/mango_selector.erl | 87 ++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/src/mango/src/mango_selector.erl b/src/mango/src/mango_selector.erl index 9c5b7a96f7f..5783ace47b6 100644 --- a/src/mango/src/mango_selector.erl +++ b/src/mango/src/mango_selector.erl @@ -1087,4 +1087,91 @@ match_beginswith_test() -> check_beginswith(<<"user_id">>, InvalidArg) ). +bench(Cases) -> + Benches = lists:map( + fun({_, Sel0, Doc}) -> + Sel = normalize(Sel0), + #{runner => fun() -> match_int(Sel, Doc) end} + end, + Cases + ), + Results = erlperf:compare(Benches, #{}), + ?debugFmt("", []), + lists:foreach( + fun({{Name, _, _}, Result}) -> + ?debugFmt("bench [~s] = ~p", [Name, Result]) + end, + lists:zip(Cases, Results) + ). + +bench_test() -> + bench([ + { + "1 field", + {[{<<"a">>, 1}]}, + {[{<<"a">>, 1}]} + }, + { + "3 sibling fields", + {[{<<"a">>, 1}, {<<"b">>, 2}, {<<"c">>, 3}]}, + {[{<<"a">>, 1}, {<<"b">>, 2}, {<<"c">>, 3}]} + }, + { + "3 nested fields", + {[{<<"a">>, {[{<<"b">>, {[{<<"c">>, 1}]}}]}}]}, + {[{<<"a">>, {[{<<"b">>, {[{<<"c">>, 1}]}}]}}]} + }, + { + "allMatch: 1 field", + {[ + {<<"a">>, + {[ + {<<"$allMatch">>, + {[ + {<<"b">>, {[{<<"$gt">>, 0}]}} + ]}} + ]}} + ]}, + {[ + {<<"a">>, [{[{<<"b">>, N}]} || N <- lists:seq(1, 10)]} + ]} + }, + { + "allMatch: 3 sibling fields", + {[ + {<<"a">>, + {[ + {<<"$allMatch">>, + {[ + {<<"b">>, {[{<<"$gt">>, 0}]}}, + {<<"c">>, {[{<<"$gt">>, 0}]}}, + {<<"d">>, {[{<<"$gt">>, 0}]}} + ]}} + ]}} + ]}, + {[ + {<<"a">>, [ + {[{<<"b">>, N}, {<<"c">>, N}, {<<"d">>, N}]} + || N <- lists:seq(1, 10) + ]} + ]} + }, + { + "allMatch: 3 nested fields", + {[ + {<<"a">>, + {[ + {<<"$allMatch">>, + {[{<<"b">>, {[{<<"c">>, {[{<<"d">>, {[{<<"$gt">>, 0}]}}]}}]}}]}} + ]}} + ]}, + {[ + {<<"a">>, [ + {[{<<"b">>, {[{<<"c">>, {[{<<"d">>, N}]}}]}}]} + || N <- lists:seq(1, 10) + ]} + ]} + } + ]). + -endif. From d339a440dcf9565735441924a7097d95b129f900 Mon Sep 17 00:00:00 2001 From: James Coglan Date: Tue, 27 Jan 2026 18:36:43 +0000 Subject: [PATCH 3/4] [wip] normalise selector fields to be pre-parsed --- src/mango/src/mango_cursor.erl | 44 +++++++-------- src/mango/src/mango_cursor_special.erl | 2 +- src/mango/src/mango_cursor_view.erl | 20 +++---- src/mango/src/mango_idx_special.erl | 15 ++++-- src/mango/src/mango_idx_view.erl | 40 ++++++++------ src/mango/src/mango_selector.erl | 75 +++++++++++++------------- src/mango/src/mango_selector_text.erl | 6 ++- src/mango/src/mango_util.erl | 21 ++++++++ 8 files changed, 131 insertions(+), 92 deletions(-) diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl index b78f3682e79..29d920ea388 100644 --- a/src/mango/src/mango_cursor.erl +++ b/src/mango/src/mango_cursor.erl @@ -286,8 +286,10 @@ extract_selector_hints(Selector) -> UnindexableFields = sets:subtract(AllFields, IndexableFields), {[ {type, IndexType}, - {indexable_fields, lists:sort(sets:to_list(IndexableFields))}, - {unindexable_fields, lists:sort(sets:to_list(UnindexableFields))} + {indexable_fields, + lists:sort([mango_util:join_field(F) || F <- sets:to_list(IndexableFields)])}, + {unindexable_fields, + lists:sort([mango_util:join_field(F) || F <- sets:to_list(UnindexableFields)])} ]} end, lists:map(Populate, Modules). @@ -358,7 +360,7 @@ explain(#cursor{} = Cursor) -> {dbname, DbName}, {index, JSON}, {partitioned, Partitioned}, - {selector, Selector}, + {selector, mango_util:join_keys(Selector)}, {opts, {Opts}}, {limit, Limit}, {skip, Skip}, @@ -1112,14 +1114,14 @@ extract_selector_hints_test_() -> t_extract_selector_hints_view(_) -> meck:expect(dreyfus, available, [], meck:val(false)), meck:expect(nouveau, enabled, [], meck:val(false)), - meck:expect(mango_selector, fields, [selector], meck:val(["field1", "field2", "field3"])), - meck:expect(mango_idx_view, indexable_fields, [selector], meck:val(["field2"])), + meck:expect(mango_selector, fields, [selector], meck:val([["field1"], ["field2"], ["field3"]])), + meck:expect(mango_idx_view, indexable_fields, [selector], meck:val([["field2"]])), Hints = [ {[ {type, json}, - {indexable_fields, ["field2"]}, - {unindexable_fields, ["field1", "field3"]} + {indexable_fields, [<<"field2">>]}, + {unindexable_fields, [<<"field1">>, <<"field3">>]} ]} ], ?assertEqual(Hints, extract_selector_hints(selector)). @@ -1127,20 +1129,20 @@ t_extract_selector_hints_view(_) -> t_extract_selector_hints_text(_) -> meck:expect(dreyfus, available, [], meck:val(true)), meck:expect(nouveau, enabled, [], meck:val(false)), - meck:expect(mango_selector, fields, [selector], meck:val(["field1", "field2", "field3"])), - meck:expect(mango_idx_view, indexable_fields, [selector], meck:val(["field2"])), - meck:expect(mango_idx_text, indexable_fields, [selector], meck:val(["field1"])), + meck:expect(mango_selector, fields, [selector], meck:val([["field1"], ["field2"], ["field3"]])), + meck:expect(mango_idx_view, indexable_fields, [selector], meck:val([["field2"]])), + meck:expect(mango_idx_text, indexable_fields, [selector], meck:val([["field1"]])), Hints = [ {[ {type, json}, - {indexable_fields, ["field2"]}, - {unindexable_fields, ["field1", "field3"]} + {indexable_fields, [<<"field2">>]}, + {unindexable_fields, [<<"field1">>, <<"field3">>]} ]}, {[ {type, text}, - {indexable_fields, ["field1"]}, - {unindexable_fields, ["field2", "field3"]} + {indexable_fields, [<<"field1">>]}, + {unindexable_fields, [<<"field2">>, <<"field3">>]} ]} ], ?assertEqual(Hints, extract_selector_hints(selector)). @@ -1148,20 +1150,20 @@ t_extract_selector_hints_text(_) -> t_extract_selector_hints_nouveau(_) -> meck:expect(dreyfus, available, [], meck:val(false)), meck:expect(nouveau, enabled, [], meck:val(true)), - meck:expect(mango_selector, fields, [selector], meck:val(["field1", "field2", "field3"])), - meck:expect(mango_idx_view, indexable_fields, [selector], meck:val(["field2"])), - meck:expect(mango_idx_nouveau, indexable_fields, [selector], meck:val(["field1"])), + meck:expect(mango_selector, fields, [selector], meck:val([["field1"], ["field2"], ["field3"]])), + meck:expect(mango_idx_view, indexable_fields, [selector], meck:val([["field2"]])), + meck:expect(mango_idx_nouveau, indexable_fields, [selector], meck:val([["field1"]])), Hints = [ {[ {type, json}, - {indexable_fields, ["field2"]}, - {unindexable_fields, ["field1", "field3"]} + {indexable_fields, [<<"field2">>]}, + {unindexable_fields, [<<"field1">>, <<"field3">>]} ]}, {[ {type, nouveau}, - {indexable_fields, ["field1"]}, - {unindexable_fields, ["field2", "field3"]} + {indexable_fields, [<<"field1">>]}, + {unindexable_fields, [<<"field2">>, <<"field3">>]} ]} ], ?assertEqual(Hints, extract_selector_hints(selector)). diff --git a/src/mango/src/mango_cursor_special.erl b/src/mango/src/mango_cursor_special.erl index c466b5db168..56f1f044361 100644 --- a/src/mango/src/mango_cursor_special.erl +++ b/src/mango/src/mango_cursor_special.erl @@ -33,7 +33,7 @@ Options :: cursor_options(). create(Db, {Indexes, Trace0}, Selector, Opts) -> InitialRange = mango_idx_view:field_ranges(Selector), - CatchAll = [{<<"_id">>, {'$gt', null, '$lt', mango_json_max}}], + CatchAll = [{[<<"_id">>], {'$gt', null, '$lt', mango_json_max}}], % order matters here - we only want to use the catchall index % if no other range can fulfill the query (because we know) % catchall is the most expensive range diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl index 0928ae19311..d9589453ebe 100644 --- a/src/mango/src/mango_cursor_view.erl +++ b/src/mango/src/mango_cursor_view.erl @@ -882,7 +882,7 @@ consider_index_coverage_positive_test() -> type = <<"json">>, def = {[{<<"fields">>, {[]}}]} }, - Fields = [<<"_id">>], + Fields = [[<<"_id">>]], MRArgs = #mrargs{ include_docs = true, @@ -1010,29 +1010,29 @@ required_fields_all_fields_test() -> ?assertEqual(all_fields, required_fields(Cursor)). required_fields_disjoint_fields_test() -> - Fields1 = [<<"field1">>, <<"field2">>, <<"field3">>], + Fields1 = [[<<"field1">>], [<<"field2">>], [<<"field3">>]], Selector1 = to_selector(#{}), Cursor1 = #cursor{fields = Fields1, selector = Selector1}, - ?assertEqual([<<"field1">>, <<"field2">>, <<"field3">>], required_fields(Cursor1)), - Fields2 = [<<"field1">>, <<"field2">>], + ?assertEqual([[<<"field1">>], [<<"field2">>], [<<"field3">>]], required_fields(Cursor1)), + Fields2 = [[<<"field1">>], [<<"field2">>]], Selector2 = to_selector(#{<<"field3">> => undefined, <<"field4">> => undefined}), - Cursor2 = #cursor{fields = Fields2, selector = to_selector(Selector2)}, + Cursor2 = #cursor{fields = Fields2, selector = Selector2}, ?assertEqual( - [<<"field1">>, <<"field2">>, <<"field3">>, <<"field4">>], required_fields(Cursor2) + [[<<"field1">>], [<<"field2">>], [<<"field3">>], [<<"field4">>]], required_fields(Cursor2) ). required_fields_overlapping_fields_test() -> - Fields1 = [<<"field1">>, <<"field2">>, <<"field3">>], + Fields1 = [[<<"field1">>], [<<"field2">>], [<<"field3">>]], Selector1 = to_selector(#{<<"field3">> => undefined, <<"field4">> => undefined}), Cursor1 = #cursor{fields = Fields1, selector = Selector1}, ?assertEqual( - [<<"field1">>, <<"field2">>, <<"field3">>, <<"field4">>], required_fields(Cursor1) + [[<<"field1">>], [<<"field2">>], [<<"field3">>], [<<"field4">>]], required_fields(Cursor1) ), - Fields2 = [<<"field3">>, <<"field1">>, <<"field2">>], + Fields2 = [[<<"field3">>], [<<"field1">>], [<<"field2">>]], Selector2 = to_selector(#{<<"field4">> => undefined, <<"field1">> => undefined}), Cursor2 = #cursor{fields = Fields2, selector = Selector2}, ?assertEqual( - [<<"field1">>, <<"field2">>, <<"field3">>, <<"field4">>], required_fields(Cursor2) + [[<<"field1">>], [<<"field2">>], [<<"field3">>], [<<"field4">>]], required_fields(Cursor2) ). explain_test() -> diff --git a/src/mango/src/mango_idx_special.erl b/src/mango/src/mango_idx_special.erl index ea3beda9c40..1be8894264f 100644 --- a/src/mango/src/mango_idx_special.erl +++ b/src/mango/src/mango_idx_special.erl @@ -55,13 +55,20 @@ to_json(#idx{def = all_docs}) -> ]}. columns(#idx{def = all_docs}) -> - [<<"_id">>]. + [[<<"_id">>]]. + +parse_field(Field) when is_binary(Field) -> + {ok, Path} = mango_util:parse_field(Field), + Path; +parse_field(Field) -> + Field. is_usable(#idx{def = all_docs}, _Selector, []) -> {true, #{reason => []}}; -is_usable(#idx{def = all_docs} = Idx, Selector, SortFields) -> +is_usable(#idx{def = all_docs} = Idx, Selector, SortFields0) -> Fields = mango_idx_view:indexable_fields(Selector), - SelectorHasRequiredFields = lists:member(<<"_id">>, Fields), + SortFields = [parse_field(F) || F <- SortFields0], + SelectorHasRequiredFields = lists:member([<<"_id">>], Fields), CanUseSort = can_use_sort(Idx, SortFields, Selector), Reason = [field_mismatch || not SelectorHasRequiredFields] ++ @@ -117,7 +124,7 @@ is_usable_test() -> SortOrderMismatch = {false, #{reason => [sort_order_mismatch]}}, Index = #idx{def = all_docs}, - ?assertEqual(FieldMismatch, usable(Index, #{}, [<<"_id">>])), + ?assertEqual(FieldMismatch, usable(Index, #{}, [[<<"_id">>]])), ?assertEqual(SortOrderMismatch, usable(Index, #{<<"_id">> => 11}, [<<"field3">>])), ?assertEqual(Usable, usable(Index, #{}, [])). -endif. diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl index e2d29200b2e..57f78765b4b 100644 --- a/src/mango/src/mango_idx_view.erl +++ b/src/mango/src/mango_idx_view.erl @@ -111,10 +111,16 @@ to_json(Idx) -> columns(Idx) -> {Props} = Idx#idx.def, {<<"fields">>, {Fields}} = lists:keyfind(<<"fields">>, 1, Props), - [Key || {Key, _} <- Fields]. + [parse_field(F) || {F, _} <- Fields]. + +parse_field(Field) when is_binary(Field) -> + {ok, Path} = mango_util:parse_field(Field), + Path; +parse_field(Field) -> + Field. -spec is_usable(#idx{}, selector(), [field()]) -> {boolean(), rejection_details()}. -is_usable(Idx, Selector, SortFields) -> +is_usable(Idx, Selector, SortFields0) -> % This index is usable if all of the columns are % restricted by the selector such that they are required to exist % and the selector is not a text search (so requires a text index) @@ -122,13 +128,14 @@ is_usable(Idx, Selector, SortFields) -> % sort fields are required to exist in the results so % we don't need to check the selector for these + SortFields = [parse_field(F) || F <- SortFields0], RequiredFields1 = ordsets:subtract(lists:usort(RequiredFields), lists:usort(SortFields)), % _id and _rev are implicitly in every document so % we don't need to check the selector for these either RequiredFields2 = ordsets:subtract( RequiredFields1, - [<<"_id">>, <<"_rev">>] + [[<<"_id">>], [<<"_rev">>]] ), SelectorHasRequiredFields = mango_selector:has_required_fields(Selector, RequiredFields2), @@ -233,7 +240,7 @@ opts() -> make_view(Idx) -> View = {[ - {<<"map">>, Idx#idx.def}, + {<<"map">>, mango_util:join_keys(Idx#idx.def)}, {<<"reduce">>, <<"_count">>}, {<<"options">>, {Idx#idx.opts}} ]}, @@ -273,7 +280,7 @@ validate_ddoc(VProps) -> % We can see through '$and' trivially indexable_fields({[{<<"$and">>, Args}]}) -> - lists:usort(lists:flatten([indexable_fields(A) || A <- Args])); + lists:usort(lists:flatmap(fun(A) -> indexable_fields(A) end, Args)); % So far we can't see through any other operator indexable_fields({[{<<"$", _/binary>>, _}]}) -> []; @@ -548,12 +555,13 @@ can_use_sort([Col | RestCols], SortFields, Selector) -> -spec covers(#idx{}, fields()) -> boolean(). covers(_, all_fields) -> false; -covers(Idx, Fields) -> +covers(Idx, Fields0) -> case mango_idx:def(Idx) of all_docs -> false; _ -> - Available = [<<"_id">> | columns(Idx)], + Fields = [parse_field(F) || F <- Fields0], + Available = [[<<"_id">>] | columns(Idx)], sets:is_subset(couch_util:set_from_list(Fields), couch_util:set_from_list(Available)) end. @@ -568,7 +576,7 @@ indexable_fields_empty_test() -> indexable_fields_and_test() -> Selector = #{<<"$and">> => [#{<<"field1">> => undefined, <<"field2">> => undefined}]}, - ?assertEqual([<<"field1">>, <<"field2">>], indexable_fields_of(Selector)). + ?assertEqual([[<<"field1">>], [<<"field2">>]], indexable_fields_of(Selector)). indexable_fields_or_test() -> Selector = #{<<"$or">> => [#{<<"field1">> => undefined, <<"field2">> => undefined}]}, @@ -608,15 +616,15 @@ indexable_fields_not_test() -> indexable_fields_lt_test() -> Selector = #{<<"field">> => #{<<"$lt">> => undefined}}, - ?assertEqual([<<"field">>], indexable_fields_of(Selector)). + ?assertEqual([[<<"field">>]], indexable_fields_of(Selector)). indexable_fields_lte_test() -> Selector = #{<<"field">> => #{<<"$lte">> => undefined}}, - ?assertEqual([<<"field">>], indexable_fields_of(Selector)). + ?assertEqual([[<<"field">>]], indexable_fields_of(Selector)). indexable_fields_eq_test() -> Selector = #{<<"field">> => #{<<"$eq">> => undefined}}, - ?assertEqual([<<"field">>], indexable_fields_of(Selector)). + ?assertEqual([[<<"field">>]], indexable_fields_of(Selector)). indexable_fields_ne_test() -> Selector = #{<<"field">> => #{<<"$ne">> => undefined}}, @@ -624,15 +632,15 @@ indexable_fields_ne_test() -> indexable_fields_gte_test() -> Selector = #{<<"field">> => #{<<"$gte">> => undefined}}, - ?assertEqual([<<"field">>], indexable_fields_of(Selector)). + ?assertEqual([[<<"field">>]], indexable_fields_of(Selector)). indexable_fields_beginswith_test() -> Selector = #{<<"field">> => #{<<"$beginsWith">> => undefined}}, - ?assertEqual([<<"field">>], indexable_fields_of(Selector)). + ?assertEqual([[<<"field">>]], indexable_fields_of(Selector)). indexable_fields_gt_test() -> Selector = #{<<"field">> => #{<<"$gt">> => undefined}}, - ?assertEqual([<<"field">>], indexable_fields_of(Selector)). + ?assertEqual([[<<"field">>]], indexable_fields_of(Selector)). indexable_fields_mod_test() -> Selector = #{<<"field">> => #{<<"$mod">> => [0, 0]}}, @@ -644,7 +652,7 @@ indexable_fields_regex_test() -> indexable_fields_exists_test() -> Selector = #{<<"field">> => #{<<"$exists">> => true}}, - ?assertEqual([<<"field">>], indexable_fields_of(Selector)). + ?assertEqual([[<<"field">>]], indexable_fields_of(Selector)). indexable_fields_type_test() -> Selector = #{<<"field">> => #{<<"$type">> => undefined}}, @@ -707,7 +715,7 @@ is_usable_test() -> ?assertEqual( SortOrderMismatch, usable(Index, #{<<"field1">> => <<"value1">>, <<"field2">> => 42}, [ - <<"field3">>, <<"field4">> + [<<"field3">>], [<<"field4">>] ]) ), ?assertEqual( diff --git a/src/mango/src/mango_selector.erl b/src/mango/src/mango_selector.erl index 5783ace47b6..569f3afbaaa 100644 --- a/src/mango/src/mango_selector.erl +++ b/src/mango/src/mango_selector.erl @@ -38,7 +38,7 @@ normalize(Selector) -> ], {NProps} = lists:foldl(fun(Step, Sel) -> Step(Sel) end, Selector, Steps), FieldNames = [Name || {Name, _} <- NProps], - case lists:member(<<>>, FieldNames) of + case lists:member([], FieldNames) of true -> ?MANGO_ERROR({invalid_selector, missing_field_name}); false -> @@ -210,7 +210,7 @@ norm_ops(Value) -> norm_fields({[]}) -> {[]}; norm_fields(Selector) -> - norm_fields(Selector, <<>>). + norm_fields(Selector, []). % Operators where we can push the field names further % down the operator tree @@ -237,7 +237,7 @@ norm_fields({[{<<"$keyMapMatch">>, Arg}]}, Path) -> % $default field. This also asserts that the $default % field is at the root as well as that it only has % a $text operator applied. -norm_fields({[{<<"$default">>, {[{<<"$text">>, _Arg}]}}]} = Sel, <<>>) -> +norm_fields({[{<<"$default">>, {[{<<"$text">>, _Arg}]}}]} = Sel, []) -> Sel; norm_fields({[{<<"$default">>, _}]} = Selector, _) -> ?MANGO_ERROR({bad_field, Selector}); @@ -249,12 +249,11 @@ norm_fields({[{<<"$", _/binary>>, _}]} = Cond, Path) -> % We've found a field name. Append it to the path % and skip this node as we unroll the stack as % the full path will be further down the branch. -norm_fields({[{Field, Cond}]}, <<>>) -> - % Don't include the '.' for the first element of - % the path. - norm_fields(Cond, Field); -norm_fields({[{Field, Cond}]}, Path) -> - norm_fields(Cond, <>); +norm_fields({[{Field, Cond}]}, Path) when is_binary(Field) -> + {ok, F} = mango_util:parse_field(Field), + norm_fields({[{F, Cond}]}, Path); +norm_fields({[{Field, Cond}]}, Path) when is_list(Field) -> + norm_fields(Cond, Path ++ Field); % An empty selector norm_fields({[]}, Path) -> {Path, {[]}}; @@ -672,7 +671,7 @@ fields({[]}) -> is_constant_field_basic_test() -> Selector = normalize({[{<<"A">>, <<"foo">>}]}), - Field = <<"A">>, + Field = [<<"A">>], ?assertEqual(true, is_constant_field(Selector, Field)). is_constant_field_basic_two_test() -> @@ -684,7 +683,7 @@ is_constant_field_basic_two_test() -> ]} ]} ), - Field = <<"cars">>, + Field = [<<"cars">>], ?assertEqual(true, is_constant_field(Selector, Field)). is_constant_field_not_eq_test() -> @@ -696,7 +695,7 @@ is_constant_field_not_eq_test() -> ]} ]} ), - Field = <<"age">>, + Field = [<<"age">>], ?assertEqual(false, is_constant_field(Selector, Field)). is_constant_field_missing_field_test() -> @@ -708,7 +707,7 @@ is_constant_field_missing_field_test() -> ]} ]} ), - Field = <<"wrong">>, + Field = [<<"wrong">>], ?assertEqual(false, is_constant_field(Selector, Field)). is_constant_field_or_field_test() -> @@ -720,12 +719,12 @@ is_constant_field_or_field_test() -> ]} ]}, Normalized = normalize(Selector), - Field = <<"A">>, + Field = [<<"A">>], ?assertEqual(false, is_constant_field(Normalized, Field)). is_constant_field_empty_selector_test() -> Selector = normalize({[]}), - Field = <<"wrong">>, + Field = [<<"wrong">>], ?assertEqual(false, is_constant_field(Selector, Field)). is_constant_nested_and_test() -> @@ -750,8 +749,8 @@ is_constant_nested_and_test() -> ]}, Normalized = normalize(Selector), - ?assertEqual(true, is_constant_field(Normalized, <<"A">>)), - ?assertEqual(false, is_constant_field(Normalized, <<"B">>)). + ?assertEqual(true, is_constant_field(Normalized, [<<"A">>])), + ?assertEqual(false, is_constant_field(Normalized, [<<"B">>])). is_constant_combined_or_and_equals_test() -> Selector = @@ -764,35 +763,35 @@ is_constant_combined_or_and_equals_test() -> {<<"C">>, "qux"} ]}, Normalized = normalize(Selector), - ?assertEqual(true, is_constant_field(Normalized, <<"C">>)), - ?assertEqual(false, is_constant_field(Normalized, <<"B">>)). + ?assertEqual(true, is_constant_field(Normalized, [<<"C">>])), + ?assertEqual(false, is_constant_field(Normalized, [<<"B">>])). has_required_fields_basic_test() -> - RequiredFields = [<<"A">>], + RequiredFields = [[<<"A">>]], Selector = {[{<<"A">>, <<"foo">>}]}, Normalized = normalize(Selector), ?assertEqual(true, has_required_fields(Normalized, RequiredFields)). has_required_fields_basic_failure_test() -> - RequiredFields = [<<"B">>], + RequiredFields = [[<<"B">>]], Selector = {[{<<"A">>, <<"foo">>}]}, Normalized = normalize(Selector), ?assertEqual(false, has_required_fields(Normalized, RequiredFields)). has_required_fields_empty_selector_test() -> - RequiredFields = [<<"A">>], + RequiredFields = [[<<"A">>]], Selector = {[]}, Normalized = normalize(Selector), ?assertEqual(false, has_required_fields(Normalized, RequiredFields)). has_required_fields_exists_false_test() -> - RequiredFields = [<<"A">>], + RequiredFields = [[<<"A">>]], Selector = {[{<<"A">>, {[{<<"$exists">>, false}]}}]}, Normalized = normalize(Selector), ?assertEqual(false, has_required_fields(Normalized, RequiredFields)). has_required_fields_and_true_test() -> - RequiredFields = [<<"A">>], + RequiredFields = [[<<"A">>]], Selector = {[ {<<"$and">>, [ @@ -804,7 +803,7 @@ has_required_fields_and_true_test() -> ?assertEqual(true, has_required_fields(Normalized, RequiredFields)). has_required_fields_nested_and_true_test() -> - RequiredFields = [<<"A">>, <<"B">>], + RequiredFields = [[<<"A">>], [<<"B">>]], Selector1 = {[ {<<"$and">>, [ @@ -829,7 +828,7 @@ has_required_fields_nested_and_true_test() -> ?assertEqual(true, has_required_fields(Normalized, RequiredFields)). has_required_fields_and_false_test() -> - RequiredFields = [<<"A">>, <<"C">>], + RequiredFields = [[<<"A">>], [<<"C">>]], Selector = {[ {<<"$and">>, [ @@ -841,7 +840,7 @@ has_required_fields_and_false_test() -> ?assertEqual(false, has_required_fields(Normalized, RequiredFields)). has_required_fields_or_false_test() -> - RequiredFields = [<<"A">>], + RequiredFields = [[<<"A">>]], Selector = {[ {<<"$or">>, [ @@ -853,7 +852,7 @@ has_required_fields_or_false_test() -> ?assertEqual(false, has_required_fields(Normalized, RequiredFields)). has_required_fields_or_true_test() -> - RequiredFields = [<<"A">>, <<"B">>, <<"C">>], + RequiredFields = [[<<"A">>], [<<"B">>], [<<"C">>]], Selector = {[ {<<"A">>, "foo"}, @@ -867,7 +866,7 @@ has_required_fields_or_true_test() -> ?assertEqual(true, has_required_fields(Normalized, RequiredFields)). has_required_fields_and_nested_or_true_test() -> - RequiredFields = [<<"A">>, <<"B">>], + RequiredFields = [[<<"A">>], [<<"B">>]], Selector1 = {[ {<<"$and">>, [ @@ -902,7 +901,7 @@ has_required_fields_and_nested_or_true_test() -> ?assertEqual(true, has_required_fields(NormalizedReverse, RequiredFields)). has_required_fields_and_nested_or_false_test() -> - RequiredFields = [<<"A">>, <<"B">>], + RequiredFields = [[<<"A">>], [<<"B">>]], Selector1 = {[ {<<"$and">>, [ @@ -938,7 +937,7 @@ has_required_fields_and_nested_or_false_test() -> ?assertEqual(false, has_required_fields(NormalizedReverse, RequiredFields)). has_required_fields_or_nested_and_true_test() -> - RequiredFields = [<<"A">>], + RequiredFields = [[<<"A">>]], Selector1 = {[ {<<"$and">>, [ @@ -962,7 +961,7 @@ has_required_fields_or_nested_and_true_test() -> ?assertEqual(true, has_required_fields(Normalized, RequiredFields)). has_required_fields_or_nested_or_true_test() -> - RequiredFields = [<<"A">>], + RequiredFields = [[<<"A">>]], Selector1 = {[ {<<"$or">>, [ @@ -986,7 +985,7 @@ has_required_fields_or_nested_or_true_test() -> ?assertEqual(true, has_required_fields(Normalized, RequiredFields)). has_required_fields_or_nested_or_false_test() -> - RequiredFields = [<<"A">>], + RequiredFields = [[<<"A">>]], Selector1 = {[ {<<"$or">>, [ @@ -1034,11 +1033,11 @@ fields_empty_test() -> fields_primitive_test() -> Selector = #{<<"field">> => undefined}, - ?assertEqual([<<"field">>], fields_of(Selector)). + ?assertEqual([[<<"field">>]], fields_of(Selector)). fields_nested_test() -> Selector = #{<<"field1">> => #{<<"field2">> => undefined}}, - ?assertEqual([<<"field1.field2">>], fields_of(Selector)). + ?assertEqual([[<<"field1">>, <<"field2">>]], fields_of(Selector)). fields_and_test() -> Selector1 = #{<<"$and">> => []}, @@ -1046,7 +1045,7 @@ fields_and_test() -> Selector2 = #{ <<"$and">> => [#{<<"field1">> => undefined}, #{<<"field2">> => undefined}] }, - ?assertEqual([<<"field1">>, <<"field2">>], fields_of(Selector2)). + ?assertEqual([[<<"field1">>], [<<"field2">>]], fields_of(Selector2)). fields_or_test() -> Selector1 = #{<<"$or">> => []}, @@ -1054,7 +1053,7 @@ fields_or_test() -> Selector2 = #{ <<"$or">> => [#{<<"field1">> => undefined}, #{<<"field2">> => undefined}] }, - ?assertEqual([<<"field1">>, <<"field2">>], fields_of(Selector2)). + ?assertEqual([[<<"field1">>], [<<"field2">>]], fields_of(Selector2)). fields_nor_test() -> Selector1 = #{<<"$nor">> => []}, @@ -1062,7 +1061,7 @@ fields_nor_test() -> Selector2 = #{ <<"$nor">> => [#{<<"field1">> => undefined}, #{<<"field2">> => undefined}] }, - ?assertEqual([<<"field1">>, <<"field2">>], fields_of(Selector2)). + ?assertEqual([[<<"field1">>], [<<"field2">>]], fields_of(Selector2)). check_beginswith(Field, Prefix) -> Selector = {[{Field, {[{<<"$beginsWith">>, Prefix}]}}]}, diff --git a/src/mango/src/mango_selector_text.erl b/src/mango/src/mango_selector_text.erl index fc30a57b614..62df9528fe8 100644 --- a/src/mango/src/mango_selector_text.erl +++ b/src/mango/src/mango_selector_text.erl @@ -179,8 +179,10 @@ convert(Path, {[{Field0, Cond}]}) -> case Field0 of <<>> -> {ok, []}; - _ -> - mango_util:parse_field(Field0) + F when is_binary(F) -> + mango_util:parse_field(F); + F when is_list(F) -> + {ok, F} end, % Later on, we perform a lucene_escape_user call on the % final Path, which calls parse_field again. Calling the function diff --git a/src/mango/src/mango_util.erl b/src/mango/src/mango_util.erl index 837cbf3dbe8..58fd2aadf87 100644 --- a/src/mango/src/mango_util.erl +++ b/src/mango/src/mango_util.erl @@ -41,6 +41,8 @@ join/2, parse_field/1, + join_field/1, + join_keys/1, cached_re/2 ]). @@ -370,6 +372,25 @@ parse_field_slow(Field) -> ), {ok, Path}. +join_keys({Sel}) when is_list(Sel) -> + Pairs = [{join_field(K), join_keys(V)} || {K, V} <- Sel], + {Pairs}; +join_keys(Sel) when is_list(Sel) -> + [join_keys(S) || S <- Sel]; +join_keys(Sel) -> + Sel. + +join_field(Field) when is_list(Field) -> + Parts = [field_to_binary(F) || F <- Field], + binary:join(Parts, <<".">>); +join_field(Field) -> + Field. + +field_to_binary(Field) when is_list(Field) -> + list_to_binary(Field); +field_to_binary(Field) when is_binary(Field) -> + Field. + check_non_empty(Field, Parts) -> case lists:member(<<>>, Parts) of true -> From 6b1d6ef2c797592ab80fad5bc2d1c1ee44c6050e Mon Sep 17 00:00:00 2001 From: James Coglan Date: Wed, 28 Jan 2026 16:21:27 +0000 Subject: [PATCH 4/4] [wip] contain the blast a bit better --- src/mango/src/mango_cursor.erl | 42 ++++++++-------- src/mango/src/mango_cursor_special.erl | 2 +- src/mango/src/mango_cursor_view.erl | 18 +++---- src/mango/src/mango_idx_special.erl | 15 ++---- src/mango/src/mango_idx_view.erl | 53 +++++++++----------- src/mango/src/mango_selector.erl | 67 ++++++++++++++------------ src/mango/src/mango_selector_text.erl | 2 - 7 files changed, 94 insertions(+), 105 deletions(-) diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl index 29d920ea388..2535a3bce4d 100644 --- a/src/mango/src/mango_cursor.erl +++ b/src/mango/src/mango_cursor.erl @@ -286,10 +286,8 @@ extract_selector_hints(Selector) -> UnindexableFields = sets:subtract(AllFields, IndexableFields), {[ {type, IndexType}, - {indexable_fields, - lists:sort([mango_util:join_field(F) || F <- sets:to_list(IndexableFields)])}, - {unindexable_fields, - lists:sort([mango_util:join_field(F) || F <- sets:to_list(UnindexableFields)])} + {indexable_fields, lists:sort(sets:to_list(IndexableFields))}, + {unindexable_fields, lists:sort(sets:to_list(UnindexableFields))} ]} end, lists:map(Populate, Modules). @@ -1114,14 +1112,14 @@ extract_selector_hints_test_() -> t_extract_selector_hints_view(_) -> meck:expect(dreyfus, available, [], meck:val(false)), meck:expect(nouveau, enabled, [], meck:val(false)), - meck:expect(mango_selector, fields, [selector], meck:val([["field1"], ["field2"], ["field3"]])), - meck:expect(mango_idx_view, indexable_fields, [selector], meck:val([["field2"]])), + meck:expect(mango_selector, fields, [selector], meck:val(["field1", "field2", "field3"])), + meck:expect(mango_idx_view, indexable_fields, [selector], meck:val(["field2"])), Hints = [ {[ {type, json}, - {indexable_fields, [<<"field2">>]}, - {unindexable_fields, [<<"field1">>, <<"field3">>]} + {indexable_fields, ["field2"]}, + {unindexable_fields, ["field1", "field3"]} ]} ], ?assertEqual(Hints, extract_selector_hints(selector)). @@ -1129,20 +1127,20 @@ t_extract_selector_hints_view(_) -> t_extract_selector_hints_text(_) -> meck:expect(dreyfus, available, [], meck:val(true)), meck:expect(nouveau, enabled, [], meck:val(false)), - meck:expect(mango_selector, fields, [selector], meck:val([["field1"], ["field2"], ["field3"]])), - meck:expect(mango_idx_view, indexable_fields, [selector], meck:val([["field2"]])), - meck:expect(mango_idx_text, indexable_fields, [selector], meck:val([["field1"]])), + meck:expect(mango_selector, fields, [selector], meck:val(["field1", "field2", "field3"])), + meck:expect(mango_idx_view, indexable_fields, [selector], meck:val(["field2"])), + meck:expect(mango_idx_text, indexable_fields, [selector], meck:val(["field1"])), Hints = [ {[ {type, json}, - {indexable_fields, [<<"field2">>]}, - {unindexable_fields, [<<"field1">>, <<"field3">>]} + {indexable_fields, ["field2"]}, + {unindexable_fields, ["field1", "field3"]} ]}, {[ {type, text}, - {indexable_fields, [<<"field1">>]}, - {unindexable_fields, [<<"field2">>, <<"field3">>]} + {indexable_fields, ["field1"]}, + {unindexable_fields, ["field2", "field3"]} ]} ], ?assertEqual(Hints, extract_selector_hints(selector)). @@ -1150,20 +1148,20 @@ t_extract_selector_hints_text(_) -> t_extract_selector_hints_nouveau(_) -> meck:expect(dreyfus, available, [], meck:val(false)), meck:expect(nouveau, enabled, [], meck:val(true)), - meck:expect(mango_selector, fields, [selector], meck:val([["field1"], ["field2"], ["field3"]])), - meck:expect(mango_idx_view, indexable_fields, [selector], meck:val([["field2"]])), - meck:expect(mango_idx_nouveau, indexable_fields, [selector], meck:val([["field1"]])), + meck:expect(mango_selector, fields, [selector], meck:val(["field1", "field2", "field3"])), + meck:expect(mango_idx_view, indexable_fields, [selector], meck:val(["field2"])), + meck:expect(mango_idx_nouveau, indexable_fields, [selector], meck:val(["field1"])), Hints = [ {[ {type, json}, - {indexable_fields, [<<"field2">>]}, - {unindexable_fields, [<<"field1">>, <<"field3">>]} + {indexable_fields, ["field2"]}, + {unindexable_fields, ["field1", "field3"]} ]}, {[ {type, nouveau}, - {indexable_fields, [<<"field1">>]}, - {unindexable_fields, [<<"field2">>, <<"field3">>]} + {indexable_fields, ["field1"]}, + {unindexable_fields, ["field2", "field3"]} ]} ], ?assertEqual(Hints, extract_selector_hints(selector)). diff --git a/src/mango/src/mango_cursor_special.erl b/src/mango/src/mango_cursor_special.erl index 56f1f044361..c466b5db168 100644 --- a/src/mango/src/mango_cursor_special.erl +++ b/src/mango/src/mango_cursor_special.erl @@ -33,7 +33,7 @@ Options :: cursor_options(). create(Db, {Indexes, Trace0}, Selector, Opts) -> InitialRange = mango_idx_view:field_ranges(Selector), - CatchAll = [{[<<"_id">>], {'$gt', null, '$lt', mango_json_max}}], + CatchAll = [{<<"_id">>, {'$gt', null, '$lt', mango_json_max}}], % order matters here - we only want to use the catchall index % if no other range can fulfill the query (because we know) % catchall is the most expensive range diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl index d9589453ebe..b496961f6dc 100644 --- a/src/mango/src/mango_cursor_view.erl +++ b/src/mango/src/mango_cursor_view.erl @@ -882,7 +882,7 @@ consider_index_coverage_positive_test() -> type = <<"json">>, def = {[{<<"fields">>, {[]}}]} }, - Fields = [[<<"_id">>]], + Fields = [<<"_id">>], MRArgs = #mrargs{ include_docs = true, @@ -1010,29 +1010,29 @@ required_fields_all_fields_test() -> ?assertEqual(all_fields, required_fields(Cursor)). required_fields_disjoint_fields_test() -> - Fields1 = [[<<"field1">>], [<<"field2">>], [<<"field3">>]], + Fields1 = [<<"field1">>, <<"field2">>, <<"field3">>], Selector1 = to_selector(#{}), Cursor1 = #cursor{fields = Fields1, selector = Selector1}, - ?assertEqual([[<<"field1">>], [<<"field2">>], [<<"field3">>]], required_fields(Cursor1)), - Fields2 = [[<<"field1">>], [<<"field2">>]], + ?assertEqual([<<"field1">>, <<"field2">>, <<"field3">>], required_fields(Cursor1)), + Fields2 = [<<"field1">>, <<"field2">>], Selector2 = to_selector(#{<<"field3">> => undefined, <<"field4">> => undefined}), Cursor2 = #cursor{fields = Fields2, selector = Selector2}, ?assertEqual( - [[<<"field1">>], [<<"field2">>], [<<"field3">>], [<<"field4">>]], required_fields(Cursor2) + [<<"field1">>, <<"field2">>, <<"field3">>, <<"field4">>], required_fields(Cursor2) ). required_fields_overlapping_fields_test() -> - Fields1 = [[<<"field1">>], [<<"field2">>], [<<"field3">>]], + Fields1 = [<<"field1">>, <<"field2">>, <<"field3">>], Selector1 = to_selector(#{<<"field3">> => undefined, <<"field4">> => undefined}), Cursor1 = #cursor{fields = Fields1, selector = Selector1}, ?assertEqual( - [[<<"field1">>], [<<"field2">>], [<<"field3">>], [<<"field4">>]], required_fields(Cursor1) + [<<"field1">>, <<"field2">>, <<"field3">>, <<"field4">>], required_fields(Cursor1) ), - Fields2 = [[<<"field3">>], [<<"field1">>], [<<"field2">>]], + Fields2 = [<<"field3">>, <<"field1">>, <<"field2">>], Selector2 = to_selector(#{<<"field4">> => undefined, <<"field1">> => undefined}), Cursor2 = #cursor{fields = Fields2, selector = Selector2}, ?assertEqual( - [[<<"field1">>], [<<"field2">>], [<<"field3">>], [<<"field4">>]], required_fields(Cursor2) + [<<"field1">>, <<"field2">>, <<"field3">>, <<"field4">>], required_fields(Cursor2) ). explain_test() -> diff --git a/src/mango/src/mango_idx_special.erl b/src/mango/src/mango_idx_special.erl index 1be8894264f..ea3beda9c40 100644 --- a/src/mango/src/mango_idx_special.erl +++ b/src/mango/src/mango_idx_special.erl @@ -55,20 +55,13 @@ to_json(#idx{def = all_docs}) -> ]}. columns(#idx{def = all_docs}) -> - [[<<"_id">>]]. - -parse_field(Field) when is_binary(Field) -> - {ok, Path} = mango_util:parse_field(Field), - Path; -parse_field(Field) -> - Field. + [<<"_id">>]. is_usable(#idx{def = all_docs}, _Selector, []) -> {true, #{reason => []}}; -is_usable(#idx{def = all_docs} = Idx, Selector, SortFields0) -> +is_usable(#idx{def = all_docs} = Idx, Selector, SortFields) -> Fields = mango_idx_view:indexable_fields(Selector), - SortFields = [parse_field(F) || F <- SortFields0], - SelectorHasRequiredFields = lists:member([<<"_id">>], Fields), + SelectorHasRequiredFields = lists:member(<<"_id">>, Fields), CanUseSort = can_use_sort(Idx, SortFields, Selector), Reason = [field_mismatch || not SelectorHasRequiredFields] ++ @@ -124,7 +117,7 @@ is_usable_test() -> SortOrderMismatch = {false, #{reason => [sort_order_mismatch]}}, Index = #idx{def = all_docs}, - ?assertEqual(FieldMismatch, usable(Index, #{}, [[<<"_id">>]])), + ?assertEqual(FieldMismatch, usable(Index, #{}, [<<"_id">>])), ?assertEqual(SortOrderMismatch, usable(Index, #{<<"_id">> => 11}, [<<"field3">>])), ?assertEqual(Usable, usable(Index, #{}, [])). -endif. diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl index 57f78765b4b..d85efe81faf 100644 --- a/src/mango/src/mango_idx_view.erl +++ b/src/mango/src/mango_idx_view.erl @@ -111,16 +111,10 @@ to_json(Idx) -> columns(Idx) -> {Props} = Idx#idx.def, {<<"fields">>, {Fields}} = lists:keyfind(<<"fields">>, 1, Props), - [parse_field(F) || {F, _} <- Fields]. - -parse_field(Field) when is_binary(Field) -> - {ok, Path} = mango_util:parse_field(Field), - Path; -parse_field(Field) -> - Field. + [Key || {Key, _} <- Fields]. -spec is_usable(#idx{}, selector(), [field()]) -> {boolean(), rejection_details()}. -is_usable(Idx, Selector, SortFields0) -> +is_usable(Idx, Selector, SortFields) -> % This index is usable if all of the columns are % restricted by the selector such that they are required to exist % and the selector is not a text search (so requires a text index) @@ -128,14 +122,13 @@ is_usable(Idx, Selector, SortFields0) -> % sort fields are required to exist in the results so % we don't need to check the selector for these - SortFields = [parse_field(F) || F <- SortFields0], RequiredFields1 = ordsets:subtract(lists:usort(RequiredFields), lists:usort(SortFields)), % _id and _rev are implicitly in every document so % we don't need to check the selector for these either RequiredFields2 = ordsets:subtract( RequiredFields1, - [[<<"_id">>], [<<"_rev">>]] + [<<"_id">>, <<"_rev">>] ), SelectorHasRequiredFields = mango_selector:has_required_fields(Selector, RequiredFields2), @@ -278,15 +271,18 @@ validate_ddoc(VProps) -> % the equivalent of a multi-query. But that's for another % day. +indexable_fields(Selector) -> + [mango_util:join_field(F) || F <- indexable_paths(Selector)]. + % We can see through '$and' trivially -indexable_fields({[{<<"$and">>, Args}]}) -> - lists:usort(lists:flatmap(fun(A) -> indexable_fields(A) end, Args)); +indexable_paths({[{<<"$and">>, Args}]}) -> + lists:usort(lists:flatmap(fun(A) -> indexable_paths(A) end, Args)); % So far we can't see through any other operator -indexable_fields({[{<<"$", _/binary>>, _}]}) -> +indexable_paths({[{<<"$", _/binary>>, _}]}) -> []; % If we have a field with a terminator that is locatable % using an index then the field is a possible index -indexable_fields({[{Field, Cond}]}) -> +indexable_paths({[{Field, Cond}]}) -> case indexable(Cond) of true -> [Field]; @@ -294,7 +290,7 @@ indexable_fields({[{Field, Cond}]}) -> [] end; % An empty selector -indexable_fields({[]}) -> +indexable_paths({[]}) -> []. % Check if a condition is indexable. The logical @@ -327,8 +323,8 @@ indexable({[{<<"$", _/binary>>, _}]}) -> % For each field, return {Field, Range} field_ranges(Selector) -> - Fields = indexable_fields(Selector), - field_ranges(Selector, Fields). + Fields = indexable_paths(Selector), + [{mango_util:join_field(F), R} || {F, R} <- field_ranges(Selector, Fields)]. field_ranges(Selector, Fields) -> field_ranges(Selector, Fields, []). @@ -555,13 +551,12 @@ can_use_sort([Col | RestCols], SortFields, Selector) -> -spec covers(#idx{}, fields()) -> boolean(). covers(_, all_fields) -> false; -covers(Idx, Fields0) -> +covers(Idx, Fields) -> case mango_idx:def(Idx) of all_docs -> false; _ -> - Fields = [parse_field(F) || F <- Fields0], - Available = [[<<"_id">>] | columns(Idx)], + Available = [<<"_id">> | columns(Idx)], sets:is_subset(couch_util:set_from_list(Fields), couch_util:set_from_list(Available)) end. @@ -576,7 +571,7 @@ indexable_fields_empty_test() -> indexable_fields_and_test() -> Selector = #{<<"$and">> => [#{<<"field1">> => undefined, <<"field2">> => undefined}]}, - ?assertEqual([[<<"field1">>], [<<"field2">>]], indexable_fields_of(Selector)). + ?assertEqual([<<"field1">>, <<"field2">>], indexable_fields_of(Selector)). indexable_fields_or_test() -> Selector = #{<<"$or">> => [#{<<"field1">> => undefined, <<"field2">> => undefined}]}, @@ -616,15 +611,15 @@ indexable_fields_not_test() -> indexable_fields_lt_test() -> Selector = #{<<"field">> => #{<<"$lt">> => undefined}}, - ?assertEqual([[<<"field">>]], indexable_fields_of(Selector)). + ?assertEqual([<<"field">>], indexable_fields_of(Selector)). indexable_fields_lte_test() -> Selector = #{<<"field">> => #{<<"$lte">> => undefined}}, - ?assertEqual([[<<"field">>]], indexable_fields_of(Selector)). + ?assertEqual([<<"field">>], indexable_fields_of(Selector)). indexable_fields_eq_test() -> Selector = #{<<"field">> => #{<<"$eq">> => undefined}}, - ?assertEqual([[<<"field">>]], indexable_fields_of(Selector)). + ?assertEqual([<<"field">>], indexable_fields_of(Selector)). indexable_fields_ne_test() -> Selector = #{<<"field">> => #{<<"$ne">> => undefined}}, @@ -632,15 +627,15 @@ indexable_fields_ne_test() -> indexable_fields_gte_test() -> Selector = #{<<"field">> => #{<<"$gte">> => undefined}}, - ?assertEqual([[<<"field">>]], indexable_fields_of(Selector)). + ?assertEqual([<<"field">>], indexable_fields_of(Selector)). indexable_fields_beginswith_test() -> Selector = #{<<"field">> => #{<<"$beginsWith">> => undefined}}, - ?assertEqual([[<<"field">>]], indexable_fields_of(Selector)). + ?assertEqual([<<"field">>], indexable_fields_of(Selector)). indexable_fields_gt_test() -> Selector = #{<<"field">> => #{<<"$gt">> => undefined}}, - ?assertEqual([[<<"field">>]], indexable_fields_of(Selector)). + ?assertEqual([<<"field">>], indexable_fields_of(Selector)). indexable_fields_mod_test() -> Selector = #{<<"field">> => #{<<"$mod">> => [0, 0]}}, @@ -652,7 +647,7 @@ indexable_fields_regex_test() -> indexable_fields_exists_test() -> Selector = #{<<"field">> => #{<<"$exists">> => true}}, - ?assertEqual([[<<"field">>]], indexable_fields_of(Selector)). + ?assertEqual([<<"field">>], indexable_fields_of(Selector)). indexable_fields_type_test() -> Selector = #{<<"field">> => #{<<"$type">> => undefined}}, @@ -715,7 +710,7 @@ is_usable_test() -> ?assertEqual( SortOrderMismatch, usable(Index, #{<<"field1">> => <<"value1">>, <<"field2">> => 42}, [ - [<<"field3">>], [<<"field4">>] + <<"field3">>, <<"field4">> ]) ), ?assertEqual( diff --git a/src/mango/src/mango_selector.erl b/src/mango/src/mango_selector.erl index 569f3afbaaa..578ea69764f 100644 --- a/src/mango/src/mango_selector.erl +++ b/src/mango/src/mango_selector.erl @@ -574,7 +574,9 @@ match({[_, _ | _] = _Props} = Sel, _Value, _Cmp) -> % match against. has_required_fields(Selector, RequiredFields) -> - Remainder = has_required_fields_int(Selector, RequiredFields), + Paths0 = [mango_util:parse_field(F) || F <- RequiredFields], + Paths = [P || {ok, P} <- Paths0], + Remainder = has_required_fields_int(Selector, Paths), Remainder == []. % Empty selector @@ -633,6 +635,9 @@ has_required_fields_int([{[{Field, Cond}]} | Rest], RequiredFields) -> end. % Returns true if a field in the selector is a constant value e.g. {a: {$eq: 1}} +is_constant_field(Selector, Field) when not is_list(Field) -> + {ok, Path} = mango_util:parse_field(Field), + is_constant_field(Selector, Path); is_constant_field({[]}, _Field) -> false; is_constant_field(Selector, Field) when not is_list(Selector) -> @@ -652,7 +657,7 @@ is_constant_field([{[{_UnMatched, _}]} | Rest], Field) -> fields({[{<<"$", _/binary>>, Args}]}) when is_list(Args) -> lists:flatmap(fun fields/1, Args); fields({[{Field, _Cond}]}) -> - [Field]; + [mango_util:join_field(Field)]; fields({[]}) -> []. @@ -671,7 +676,7 @@ fields({[]}) -> is_constant_field_basic_test() -> Selector = normalize({[{<<"A">>, <<"foo">>}]}), - Field = [<<"A">>], + Field = <<"A">>, ?assertEqual(true, is_constant_field(Selector, Field)). is_constant_field_basic_two_test() -> @@ -683,7 +688,7 @@ is_constant_field_basic_two_test() -> ]} ]} ), - Field = [<<"cars">>], + Field = <<"cars">>, ?assertEqual(true, is_constant_field(Selector, Field)). is_constant_field_not_eq_test() -> @@ -695,7 +700,7 @@ is_constant_field_not_eq_test() -> ]} ]} ), - Field = [<<"age">>], + Field = <<"age">>, ?assertEqual(false, is_constant_field(Selector, Field)). is_constant_field_missing_field_test() -> @@ -707,7 +712,7 @@ is_constant_field_missing_field_test() -> ]} ]} ), - Field = [<<"wrong">>], + Field = <<"wrong">>, ?assertEqual(false, is_constant_field(Selector, Field)). is_constant_field_or_field_test() -> @@ -719,12 +724,12 @@ is_constant_field_or_field_test() -> ]} ]}, Normalized = normalize(Selector), - Field = [<<"A">>], + Field = <<"A">>, ?assertEqual(false, is_constant_field(Normalized, Field)). is_constant_field_empty_selector_test() -> Selector = normalize({[]}), - Field = [<<"wrong">>], + Field = <<"wrong">>, ?assertEqual(false, is_constant_field(Selector, Field)). is_constant_nested_and_test() -> @@ -749,8 +754,8 @@ is_constant_nested_and_test() -> ]}, Normalized = normalize(Selector), - ?assertEqual(true, is_constant_field(Normalized, [<<"A">>])), - ?assertEqual(false, is_constant_field(Normalized, [<<"B">>])). + ?assertEqual(true, is_constant_field(Normalized, <<"A">>)), + ?assertEqual(false, is_constant_field(Normalized, <<"B">>)). is_constant_combined_or_and_equals_test() -> Selector = @@ -763,35 +768,35 @@ is_constant_combined_or_and_equals_test() -> {<<"C">>, "qux"} ]}, Normalized = normalize(Selector), - ?assertEqual(true, is_constant_field(Normalized, [<<"C">>])), - ?assertEqual(false, is_constant_field(Normalized, [<<"B">>])). + ?assertEqual(true, is_constant_field(Normalized, <<"C">>)), + ?assertEqual(false, is_constant_field(Normalized, <<"B">>)). has_required_fields_basic_test() -> - RequiredFields = [[<<"A">>]], + RequiredFields = [<<"A">>], Selector = {[{<<"A">>, <<"foo">>}]}, Normalized = normalize(Selector), ?assertEqual(true, has_required_fields(Normalized, RequiredFields)). has_required_fields_basic_failure_test() -> - RequiredFields = [[<<"B">>]], + RequiredFields = [<<"B">>], Selector = {[{<<"A">>, <<"foo">>}]}, Normalized = normalize(Selector), ?assertEqual(false, has_required_fields(Normalized, RequiredFields)). has_required_fields_empty_selector_test() -> - RequiredFields = [[<<"A">>]], + RequiredFields = [<<"A">>], Selector = {[]}, Normalized = normalize(Selector), ?assertEqual(false, has_required_fields(Normalized, RequiredFields)). has_required_fields_exists_false_test() -> - RequiredFields = [[<<"A">>]], + RequiredFields = [<<"A">>], Selector = {[{<<"A">>, {[{<<"$exists">>, false}]}}]}, Normalized = normalize(Selector), ?assertEqual(false, has_required_fields(Normalized, RequiredFields)). has_required_fields_and_true_test() -> - RequiredFields = [[<<"A">>]], + RequiredFields = [<<"A">>], Selector = {[ {<<"$and">>, [ @@ -803,7 +808,7 @@ has_required_fields_and_true_test() -> ?assertEqual(true, has_required_fields(Normalized, RequiredFields)). has_required_fields_nested_and_true_test() -> - RequiredFields = [[<<"A">>], [<<"B">>]], + RequiredFields = [<<"A">>, <<"B">>], Selector1 = {[ {<<"$and">>, [ @@ -828,7 +833,7 @@ has_required_fields_nested_and_true_test() -> ?assertEqual(true, has_required_fields(Normalized, RequiredFields)). has_required_fields_and_false_test() -> - RequiredFields = [[<<"A">>], [<<"C">>]], + RequiredFields = [<<"A">>, <<"C">>], Selector = {[ {<<"$and">>, [ @@ -840,7 +845,7 @@ has_required_fields_and_false_test() -> ?assertEqual(false, has_required_fields(Normalized, RequiredFields)). has_required_fields_or_false_test() -> - RequiredFields = [[<<"A">>]], + RequiredFields = [<<"A">>], Selector = {[ {<<"$or">>, [ @@ -852,7 +857,7 @@ has_required_fields_or_false_test() -> ?assertEqual(false, has_required_fields(Normalized, RequiredFields)). has_required_fields_or_true_test() -> - RequiredFields = [[<<"A">>], [<<"B">>], [<<"C">>]], + RequiredFields = [<<"A">>, <<"B">>, <<"C">>], Selector = {[ {<<"A">>, "foo"}, @@ -866,7 +871,7 @@ has_required_fields_or_true_test() -> ?assertEqual(true, has_required_fields(Normalized, RequiredFields)). has_required_fields_and_nested_or_true_test() -> - RequiredFields = [[<<"A">>], [<<"B">>]], + RequiredFields = [<<"A">>, <<"B">>], Selector1 = {[ {<<"$and">>, [ @@ -901,7 +906,7 @@ has_required_fields_and_nested_or_true_test() -> ?assertEqual(true, has_required_fields(NormalizedReverse, RequiredFields)). has_required_fields_and_nested_or_false_test() -> - RequiredFields = [[<<"A">>], [<<"B">>]], + RequiredFields = [<<"A">>, <<"B">>], Selector1 = {[ {<<"$and">>, [ @@ -937,7 +942,7 @@ has_required_fields_and_nested_or_false_test() -> ?assertEqual(false, has_required_fields(NormalizedReverse, RequiredFields)). has_required_fields_or_nested_and_true_test() -> - RequiredFields = [[<<"A">>]], + RequiredFields = [<<"A">>], Selector1 = {[ {<<"$and">>, [ @@ -961,7 +966,7 @@ has_required_fields_or_nested_and_true_test() -> ?assertEqual(true, has_required_fields(Normalized, RequiredFields)). has_required_fields_or_nested_or_true_test() -> - RequiredFields = [[<<"A">>]], + RequiredFields = [<<"A">>], Selector1 = {[ {<<"$or">>, [ @@ -985,7 +990,7 @@ has_required_fields_or_nested_or_true_test() -> ?assertEqual(true, has_required_fields(Normalized, RequiredFields)). has_required_fields_or_nested_or_false_test() -> - RequiredFields = [[<<"A">>]], + RequiredFields = [<<"A">>], Selector1 = {[ {<<"$or">>, [ @@ -1033,11 +1038,11 @@ fields_empty_test() -> fields_primitive_test() -> Selector = #{<<"field">> => undefined}, - ?assertEqual([[<<"field">>]], fields_of(Selector)). + ?assertEqual([<<"field">>], fields_of(Selector)). fields_nested_test() -> Selector = #{<<"field1">> => #{<<"field2">> => undefined}}, - ?assertEqual([[<<"field1">>, <<"field2">>]], fields_of(Selector)). + ?assertEqual([<<"field1.field2">>], fields_of(Selector)). fields_and_test() -> Selector1 = #{<<"$and">> => []}, @@ -1045,7 +1050,7 @@ fields_and_test() -> Selector2 = #{ <<"$and">> => [#{<<"field1">> => undefined}, #{<<"field2">> => undefined}] }, - ?assertEqual([[<<"field1">>], [<<"field2">>]], fields_of(Selector2)). + ?assertEqual([<<"field1">>, <<"field2">>], fields_of(Selector2)). fields_or_test() -> Selector1 = #{<<"$or">> => []}, @@ -1053,7 +1058,7 @@ fields_or_test() -> Selector2 = #{ <<"$or">> => [#{<<"field1">> => undefined}, #{<<"field2">> => undefined}] }, - ?assertEqual([[<<"field1">>], [<<"field2">>]], fields_of(Selector2)). + ?assertEqual([<<"field1">>, <<"field2">>], fields_of(Selector2)). fields_nor_test() -> Selector1 = #{<<"$nor">> => []}, @@ -1061,7 +1066,7 @@ fields_nor_test() -> Selector2 = #{ <<"$nor">> => [#{<<"field1">> => undefined}, #{<<"field2">> => undefined}] }, - ?assertEqual([[<<"field1">>], [<<"field2">>]], fields_of(Selector2)). + ?assertEqual([<<"field1">>, <<"field2">>], fields_of(Selector2)). check_beginswith(Field, Prefix) -> Selector = {[{Field, {[{<<"$beginsWith">>, Prefix}]}}]}, diff --git a/src/mango/src/mango_selector_text.erl b/src/mango/src/mango_selector_text.erl index 62df9528fe8..8b5152dd151 100644 --- a/src/mango/src/mango_selector_text.erl +++ b/src/mango/src/mango_selector_text.erl @@ -177,8 +177,6 @@ convert(_Path, {[{<<"$", _/binary>> = Op, _}]}) -> convert(Path, {[{Field0, Cond}]}) -> {ok, PP0} = case Field0 of - <<>> -> - {ok, []}; F when is_binary(F) -> mango_util:parse_field(F); F when is_list(F) ->