diff --git a/.gitignore b/.gitignore index 28dcadf05f7..cf6e1ccc4f0 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ share/server/main-coffee.js share/server/main.js share/server/main-ast-bypass.js share/www +src/argparse/ src/bear/ src/certifi/ src/couch/priv/couch_js/**/config.h @@ -55,6 +56,7 @@ src/couch/priv/couch_js/**/*.d src/couch/priv/icu_driver/couch_icu_driver.d src/cowlib/ src/mango/src/mango_cursor_text.nocompile +src/erlperf/ src/excoveralls/ src/fauxton/ src/folsom/ diff --git a/rebar.config.script b/rebar.config.script index efc03a35e64..91c91f4a1dd 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -164,7 +164,9 @@ DepDescs = [ {jiffy, "jiffy", {tag, "1.1.2"}}, {mochiweb, "mochiweb", {tag, "v3.3.0"}}, {meck, "meck", {tag, "v1.1.0"}}, -{recon, "recon", {tag, "2.5.6"}} +{recon, "recon", {tag, "2.5.6"}}, +{argparse, {url, "https://github.com/max-au/argparse"}, "1.2.4"}, +{erlperf, {url, "https://github.com/max-au/erlperf"}, "2.3.0"} ]. WithProper = lists:keyfind(with_proper, 1, CouchConfig) == {with_proper, true}. diff --git a/src/couch/src/couch_query_servers.erl b/src/couch/src/couch_query_servers.erl index 1652fc09a2a..5d7e9ac1fab 100644 --- a/src/couch/src/couch_query_servers.erl +++ b/src/couch/src/couch_query_servers.erl @@ -477,19 +477,21 @@ builtin_cmp_last(A, B) -> validate_doc_update(Db, DDoc, EditDoc, DiskDoc, Ctx, SecObj) -> JsonEditDoc = couch_doc:to_json_obj(EditDoc, [revs]), JsonDiskDoc = json_doc(DiskDoc), - Resp = ddoc_prompt( - Db, - DDoc, - [<<"validate_doc_update">>], - [JsonEditDoc, JsonDiskDoc, Ctx, SecObj] - ), - if - Resp == 1 -> ok; - true -> couch_stats:increment_counter([couchdb, query_server, vdu_rejects], 1) - end, + Args = [JsonEditDoc, JsonDiskDoc, Ctx, SecObj], + + Resp = + case ddoc_prompt(Db, DDoc, [<<"validate_doc_update">>], Args) of + Code when Code =:= 1; Code =:= ok; Code =:= true -> + ok; + Other -> + couch_stats:increment_counter([couchdb, query_server, vdu_rejects], 1), + Other + end, case Resp of - RespCode when RespCode =:= 1; RespCode =:= ok; RespCode =:= true -> + ok -> ok; + {[{<<"forbidden">>, Message}, {<<"failures">>, Failures}]} -> + throw({forbidden, Message, Failures}); {[{<<"forbidden">>, Message}]} -> throw({forbidden, Message}); {[{<<"unauthorized">>, Message}]} -> diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl index bc7b1f8abf3..244f668af03 100644 --- a/src/couch_mrview/src/couch_mrview.erl +++ b/src/couch_mrview/src/couch_mrview.erl @@ -62,7 +62,7 @@ validate_ddoc_fields(DDoc) -> [{<<"rewrites">>, [string, array]}], [{<<"shows">>, object}, {any, [object, string]}], [{<<"updates">>, object}, {any, [object, string]}], - [{<<"validate_doc_update">>, string}], + [{<<"validate_doc_update">>, [string, object]}], [{<<"views">>, object}, {<<"lib">>, object}], [{<<"views">>, object}, {any, object}, {<<"map">>, MapFuncType}], [{<<"views">>, object}, {any, object}, {<<"reduce">>, string}] diff --git a/src/mango/src/mango_doc.erl b/src/mango/src/mango_doc.erl index 255debf9e6d..8093c6f7292 100644 --- a/src/mango/src/mango_doc.erl +++ b/src/mango/src/mango_doc.erl @@ -21,6 +21,8 @@ get_field/2, get_field/3, + get_field_with_stack/3, + get_field_from_stack/2, rem_field/2, set_field/3 ]). @@ -377,26 +379,31 @@ do_update_to_insert([{_, _} | Rest], Doc) -> get_field(Props, Field) -> get_field(Props, Field, no_validation). -get_field(Props, Field, Validator) when is_binary(Field) -> - {ok, Path} = mango_util:parse_field(Field), - get_field(Props, Path, Validator); -get_field(Props, [], no_validation) -> - Props; -get_field(Props, [], Validator) -> - case (catch Validator(Props)) of +get_field(Props, Field, no_validation) -> + {Value, _} = get_field_with_stack(Props, Field, []), + Value; +get_field(Props, Field, Validator) -> + {Value, _} = get_field_with_stack(Props, Field, []), + case (catch Validator(Value)) of true -> - Props; + Value; _ -> invalid_value - end; -get_field({Props}, [Name | Rest], Validator) -> + end. + +get_field_with_stack(Props, Field, Stack) when is_binary(Field) -> + {ok, Path} = mango_util:parse_field(Field), + get_field_with_stack(Props, Path, Stack); +get_field_with_stack(Props, [], Stack) -> + {Props, Stack}; +get_field_with_stack({Props}, [Name | Rest], Stack) -> case lists:keyfind(Name, 1, Props) of {Name, Value} -> - get_field(Value, Rest, Validator); + get_field_with_stack(Value, Rest, [{Props} | Stack]); false -> - not_found + {not_found, [{Props} | Stack]} end; -get_field(Values, [Name | Rest], Validator) when is_list(Values) -> +get_field_with_stack(Values, [Name | Rest], Stack) when is_list(Values) -> % Name might be an integer index into an array try Pos = binary_to_integer(Name), @@ -404,16 +411,24 @@ get_field(Values, [Name | Rest], Validator) when is_list(Values) -> true -> % +1 because Erlang uses 1 based list indices Value = lists:nth(Pos + 1, Values), - get_field(Value, Rest, Validator); + get_field_with_stack(Value, Rest, Stack); false -> - bad_path + {bad_path, Stack} end catch error:badarg -> - bad_path + {bad_path, Stack} end; -get_field(_, [_ | _], _) -> - bad_path. +get_field_with_stack(_, [_ | _], Stack) -> + {bad_path, Stack}. + +get_field_from_stack([{[{<<"parent">>, N}]} | Rest], Stack) -> + case length(Stack) >= N of + true -> get_field(lists:nth(N, Stack), Rest); + false -> not_found + end; +get_field_from_stack(Path, Stack) -> + get_field(lists:last(Stack), Path). rem_field(Props, Field) when is_binary(Field) -> {ok, Path} = mango_util:parse_field(Field), diff --git a/src/mango/src/mango_native_proc.erl b/src/mango/src/mango_native_proc.erl index 511a987199e..b540047e509 100644 --- a/src/mango/src/mango_native_proc.erl +++ b/src/mango/src/mango_native_proc.erl @@ -29,6 +29,7 @@ -record(st, { indexes = [], + validators = [], timeout = 5000 }). @@ -94,6 +95,37 @@ handle_call({prompt, [<<"nouveau_index_doc">>, Doc]}, _From, St) -> Else end, {reply, Vals, St}; +handle_call({prompt, [<<"ddoc">>, <<"new">>, DDocId, {DDoc}]}, _From, St) -> + NewSt = + case couch_util:get_value(<<"validate_doc_update">>, DDoc) of + undefined -> + St; + Selector0 -> + Selector = mango_selector:normalize(Selector0), + Validators = couch_util:set_value(DDocId, St#st.validators, Selector), + St#st{validators = Validators} + end, + {reply, true, NewSt}; +handle_call({prompt, [<<"ddoc">>, DDocId, [<<"validate_doc_update">>], Args]}, _From, St) -> + case couch_util:get_value(DDocId, St#st.validators) of + undefined -> + Msg = [<<"validate_doc_update">>, DDocId], + {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}; + Selector -> + [NewDoc, OldDoc, _Ctx, _SecObj] = Args, + Struct = {[{<<"newDoc">>, NewDoc}, {<<"oldDoc">>, OldDoc}]}, + Reply = + case mango_selector:match_failures(Selector, Struct) of + [] -> + true; + Failures -> + {[ + {<<"forbidden">>, <<"forbidden">>}, + {<<"failures">>, Failures} + ]} + end, + {reply, Reply, St} + end; handle_call(Msg, _From, St) -> {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}. diff --git a/src/mango/src/mango_selector.erl b/src/mango/src/mango_selector.erl index 9c5b7a96f7f..b8b78f69177 100644 --- a/src/mango/src/mango_selector.erl +++ b/src/mango/src/mango_selector.erl @@ -15,6 +15,7 @@ -export([ normalize/1, match/2, + match_failures/2, has_required_fields/2, is_constant_field/2, fields/1 @@ -23,6 +24,21 @@ -include_lib("couch/include/couch_db.hrl"). -include("mango.hrl"). +-record(ctx, { + cmp, + verbose = false, + negate = false, + path = [], + stack = [] +}). + +-record(failure, { + op, + type = mismatch, + params = [], + ctx +}). + % Validate and normalize each operator. This translates % every selector operator into a consistent version that % we can then rely on for all other selector functions. @@ -53,17 +69,30 @@ match(Selector, D) -> couch_stats:increment_counter([mango, evaluate_selector]), match_int(Selector, D). -% An empty selector matches any value. -match_int({[]}, _) -> - true; -match_int(Selector, #doc{body = Body}) -> - match(Selector, Body, fun mango_json:cmp/2); -match_int(Selector, {Props}) -> - match(Selector, {Props}, fun mango_json:cmp/2). +match_failures(Selector, D) -> + couch_stats:increment_counter([mango, evaluate_selector]), + [format_failure(F) || F <- match_int(Selector, D, true)]. + +match_int(Selector, D) -> + match_int(Selector, D, false). + +match_int(Selector, D, Verbose) -> + Ctx = #ctx{cmp = fun mango_json:cmp/2, verbose = Verbose}, + case D of + #doc{body = Body} -> match(Selector, Body, Ctx); + Other -> match(Selector, Other, Ctx) + end. % Convert each operator into a normalized version as well % as convert an implicit operators into their explicit % versions. +% {$Op: {$data: Path}} +norm_ops({[{<<"$", _/binary>>, {[{<<"$data">>, Path}]}}]} = Cond) when is_binary(Path) -> + norm_data(Cond); +% {Field: {$data: Path}} +norm_ops({[{Field, {[{<<"$data">>, Path}]}}]}) when is_binary(Path) -> + Eq = norm_data({[{<<"$eq">>, {[{<<"$data">>, Path}]}}]}), + {[{Field, Eq}]}; norm_ops({[{<<"$and">>, Args}]}) when is_list(Args) -> {[{<<"$and">>, [norm_ops(A) || A <- Args]}]}; norm_ops({[{<<"$and">>, Arg}]}) -> @@ -107,8 +136,8 @@ norm_ops({[{<<"$regex">>, Regex}]} = Cond) when is_binary(Regex) -> _ -> ?MANGO_ERROR({bad_arg, '$regex', Regex}) end; -norm_ops({[{<<"$all">>, Args}]}) when is_list(Args) -> - {[{<<"$all">>, Args}]}; +norm_ops({[{<<"$all">>, Args}]} = Cond) when is_list(Args) -> + Cond; norm_ops({[{<<"$all">>, Arg}]}) -> ?MANGO_ERROR({bad_arg, '$all', Arg}); norm_ops({[{<<"$elemMatch">>, {_} = Arg}]}) -> @@ -123,8 +152,8 @@ norm_ops({[{<<"$keyMapMatch">>, {_} = Arg}]}) -> {[{<<"$keyMapMatch">>, norm_ops(Arg)}]}; norm_ops({[{<<"$keyMapMatch">>, Arg}]}) -> ?MANGO_ERROR({bad_arg, '$keyMapMatch', Arg}); -norm_ops({[{<<"$size">>, Arg}]}) when is_integer(Arg), Arg >= 0 -> - {[{<<"$size">>, Arg}]}; +norm_ops({[{<<"$size">>, Arg}]} = Cond) when is_integer(Arg), Arg >= 0 -> + Cond; norm_ops({[{<<"$size">>, Arg}]}) -> ?MANGO_ERROR({bad_arg, '$size', Arg}); norm_ops({[{<<"$text">>, Arg}]}) when @@ -184,6 +213,37 @@ norm_ops({[_, _ | _] = Props}) -> norm_ops(Value) -> {[{<<"$eq">>, Value}]}. +% {$data: Path} may only be used as an argument to "leaf" operators that expect +% a literal value as input. If it were combined with combinators like $and or +% $allMatch it would allow the input document to inject its own selectors. +-define(DATA_OPS, [ + <<"$eq">>, + <<"$ne">>, + <<"$lt">>, + <<"$lte">>, + <<"$gt">>, + <<"$gte">>, + <<"$in">>, + <<"$nin">>, + <<"$all">>, + <<"$type">>, + <<"$size">>, + <<"$mod">>, + <<"$regex">>, + <<"$beginsWith">> +]). + +norm_data({[{Op, {[{<<"$data">>, Field}]}}]}) when is_binary(Field) -> + case lists:member(Op, ?DATA_OPS) of + true -> + {ok, Path} = mango_util:parse_field(Field, relative), + {[{Op, {[{<<"$data">>, Path}]}}]}; + false -> + ?MANGO_ERROR({bad_arg, '$data', Op}) + end; +norm_data({[{_, _}]} = Cond) -> + Cond. + % This takes a selector and normalizes all of the % field names as far as possible. For instance: % @@ -362,26 +422,78 @@ negate({[{<<"$", _/binary>>, _}]} = Cond) -> negate({[{Field, Cond}]}) -> {[{Field, negate(Cond)}]}. +% An empty selector matches any value. +match({[]}, _, #ctx{verbose = false}) -> + true; +match({[]}, _, #ctx{verbose = true}) -> + []; +% Resolve $data lookups before evaluating the surrounding operator +match({[{Op, {[{<<"$data">>, Path}]}}]}, Value, #ctx{stack = Stack, verbose = Verbose} = Ctx) -> + case mango_doc:get_field_from_stack(Path, Stack) of + not_found -> + case Verbose of + true -> [#failure{op = data, type = not_found, ctx = Ctx#ctx{path = Path}}]; + false -> false + end; + bad_path -> + case Verbose of + true -> [#failure{op = data, type = bad_path, ctx = Ctx#ctx{path = Path}}]; + false -> false + end; + Found -> + match({[{Op, Found}]}, Value, Ctx) + end; % We need to treat an empty array as always true. This will be applied % for $or, $in, $all, $nin as well. -match({[{<<"$and">>, []}]}, _, _) -> +match({[{<<"$and">>, []}]}, _, #ctx{verbose = false}) -> true; -match({[{<<"$and">>, Args}]}, Value, Cmp) -> - Pred = fun(SubSel) -> match(SubSel, Value, Cmp) end, +match({[{<<"$and">>, []}]}, _, #ctx{negate = false}) -> + []; +match({[{<<"$and">>, []}]}, _, Ctx) -> + [#failure{op = 'and', type = empty_list, params = [[]], ctx = Ctx}]; +match({[{<<"$and">>, Args}]}, Value, #ctx{verbose = false} = Ctx) -> + Pred = fun(SubSel) -> match(SubSel, Value, Ctx) end, lists:all(Pred, Args); -match({[{<<"$or">>, []}]}, _, _) -> +match({[{<<"$and">>, Args}]}, Value, #ctx{negate = true} = Ctx) -> + NotArgs = [{[{<<"$not">>, A}]} || A <- Args], + PosCtx = Ctx#ctx{negate = false}, + match({[{<<"$or">>, NotArgs}]}, Value, PosCtx); +match({[{<<"$and">>, Args}]}, Value, Ctx) -> + MatchSubSel = fun(SubSel) -> match(SubSel, Value, Ctx) end, + lists:flatmap(MatchSubSel, Args); +match({[{<<"$or">>, []}]}, _, #ctx{verbose = false}) -> true; -match({[{<<"$or">>, Args}]}, Value, Cmp) -> - Pred = fun(SubSel) -> match(SubSel, Value, Cmp) end, +match({[{<<"$or">>, []}]}, _, #ctx{negate = false}) -> + []; +match({[{<<"$or">>, []}]}, _, Ctx) -> + [#failure{op = 'or', type = empty_list, params = [[]], ctx = Ctx}]; +match({[{<<"$or">>, Args}]}, Value, #ctx{verbose = false} = Ctx) -> + Pred = fun(SubSel) -> match(SubSel, Value, Ctx) end, lists:any(Pred, Args); -match({[{<<"$not">>, Arg}]}, Value, Cmp) -> - not match(Arg, Value, Cmp); -match({[{<<"$all">>, []}]}, _, _) -> +match({[{<<"$or">>, Args}]}, Value, #ctx{negate = true} = Ctx) -> + NotArgs = [{[{<<"$not">>, A}]} || A <- Args], + PosCtx = Ctx#ctx{negate = false}, + match({[{<<"$and">>, NotArgs}]}, Value, PosCtx); +match({[{<<"$or">>, Args}]}, Value, Ctx) -> + SubSelFailures = [match(A, Value, Ctx) || A <- Args], + case lists:member([], SubSelFailures) of + true -> []; + false -> lists:flatten(SubSelFailures) + end; +match({[{<<"$not">>, Arg}]}, Value, #ctx{verbose = false} = Ctx) -> + not match(Arg, Value, Ctx); +match({[{<<"$not">>, Arg}]}, Value, #ctx{negate = Neg} = Ctx) -> + match(Arg, Value, Ctx#ctx{negate = not Neg}); +match({[{<<"$all">>, []}]}, _, #ctx{verbose = false}) -> false; +match({[{<<"$all">>, []}]}, _, #ctx{negate = false} = Ctx) -> + [#failure{op = all, type = empty_list, params = [[]], ctx = Ctx}]; +match({[{<<"$all">>, []}]}, _, #ctx{negate = true}) -> + []; % All of the values in Args must exist in Values or % Values == hd(Args) if Args is a single element list % that contains a list. -match({[{<<"$all">>, Args}]}, Values, _Cmp) when is_list(Values) -> +match({[{<<"$all">>, Args}]}, Values, #ctx{verbose = false}) when is_list(Values) -> Pred = fun(A) -> lists:member(A, Values) end, HasArgs = lists:all(Pred, Args), IsArgs = @@ -392,8 +504,10 @@ match({[{<<"$all">>, Args}]}, Values, _Cmp) when is_list(Values) -> false end, HasArgs orelse IsArgs; -match({[{<<"$all">>, _Args}]}, _Values, _Cmp) -> +match({[{<<"$all">>, _Args}]}, _Values, #ctx{verbose = false}) -> false; +match({[{<<"$all">>, Args}]} = Expr, Values, Ctx) -> + match_with_failure(Expr, Values, all, [Args], Ctx); %% This is for $elemMatch, $allMatch, and possibly $in because of our normalizer. %% A selector such as {"field_name": {"$elemMatch": {"$gte": 80, "$lt": 85}}} %% gets normalized to: @@ -406,15 +520,15 @@ match({[{<<"$all">>, _Args}]}, _Values, _Cmp) -> %% }]} %% }]}. %% So we filter out the <<>>. -match({[{<<>>, Arg}]}, Values, Cmp) -> - match(Arg, Values, Cmp); +match({[{<<>>, Arg}]}, Values, Ctx) -> + match(Arg, Values, Ctx); % Matches when any element in values matches the % sub-selector Arg. -match({[{<<"$elemMatch">>, Arg}]}, Values, Cmp) when is_list(Values) -> +match({[{<<"$elemMatch">>, Arg}]}, Values, #ctx{verbose = false} = Ctx) when is_list(Values) -> try lists:foreach( fun(V) -> - case match(Arg, V, Cmp) of + case match(Arg, V, Ctx) of true -> throw(matched); _ -> ok end @@ -428,15 +542,31 @@ match({[{<<"$elemMatch">>, Arg}]}, Values, Cmp) when is_list(Values) -> _:_ -> false end; -match({[{<<"$elemMatch">>, _Arg}]}, _Value, _Cmp) -> +match({[{<<"$elemMatch">>, _Arg}]}, _Value, #ctx{verbose = false}) -> false; +match({[{<<"$elemMatch">>, _Arg}]}, [], #ctx{negate = false} = Ctx) -> + [#failure{op = elemMatch, type = empty_list, ctx = Ctx}]; +match({[{<<"$elemMatch">>, _Arg}]}, [], #ctx{negate = true}) -> + []; +match({[{<<"$elemMatch">>, Arg}]}, Values, #ctx{negate = true} = Ctx) -> + PosCtx = Ctx#ctx{negate = false}, + match({[{<<"$allMatch">>, {[{<<"$not">>, Arg}]}}]}, Values, PosCtx); +match({[{<<"$elemMatch">>, Arg}]}, Values, #ctx{path = Path} = Ctx) -> + ValueFailures = [ + match(Arg, V, Ctx#ctx{path = [Idx | Path]}) + || {Idx, V} <- lists:enumerate(0, Values) + ], + case lists:member([], ValueFailures) of + true -> []; + false -> lists:flatten(ValueFailures) + end; % Matches when all elements in values match the % sub-selector Arg. -match({[{<<"$allMatch">>, Arg}]}, [_ | _] = Values, Cmp) -> +match({[{<<"$allMatch">>, Arg}]}, [_ | _] = Values, #ctx{verbose = false} = Ctx) -> try lists:foreach( fun(V) -> - case match(Arg, V, Cmp) of + case match(Arg, V, Ctx) of false -> throw(unmatched); _ -> ok end @@ -448,15 +578,26 @@ match({[{<<"$allMatch">>, Arg}]}, [_ | _] = Values, Cmp) -> _:_ -> false end; -match({[{<<"$allMatch">>, _Arg}]}, _Value, _Cmp) -> +match({[{<<"$allMatch">>, _Arg}]}, _Value, #ctx{verbose = false}) -> false; +match({[{<<"$allMatch">>, _Arg}]}, [], #ctx{negate = false} = Ctx) -> + [#failure{op = allMatch, type = empty_list, ctx = Ctx}]; +match({[{<<"$allMatch">>, _Arg}]}, [], #ctx{negate = true}) -> + []; +match({[{<<"$allMatch">>, Arg}]}, Values, #ctx{negate = true} = Ctx) -> + PosCtx = Ctx#ctx{negate = false}, + match({[{<<"$elemMatch">>, {[{<<"$not">>, Arg}]}}]}, Values, PosCtx); +match({[{<<"$allMatch">>, Arg}]}, Values, #ctx{path = Path} = Ctx) -> + MatchValue = fun({Idx, V}) -> match(Arg, V, Ctx#ctx{path = [Idx | Path]}) end, + EnumValues = lists:enumerate(0, Values), + lists:flatmap(MatchValue, EnumValues); % Matches when any key in the map value matches the % sub-selector Arg. -match({[{<<"$keyMapMatch">>, Arg}]}, Value, Cmp) when is_tuple(Value) -> +match({[{<<"$keyMapMatch">>, Arg}]}, Value, #ctx{verbose = false} = Ctx) when is_tuple(Value) -> try lists:foreach( fun(V) -> - case match(Arg, V, Cmp) of + case match(Arg, V, Ctx) of true -> throw(matched); _ -> ok end @@ -470,24 +611,43 @@ match({[{<<"$keyMapMatch">>, Arg}]}, Value, Cmp) when is_tuple(Value) -> _:_ -> false end; -match({[{<<"$keyMapMatch">>, _Arg}]}, _Value, _Cmp) -> +match({[{<<"$keyMapMatch">>, _Arg}]}, _Value, #ctx{verbose = false}) -> false; +match({[{<<"$keyMapMatch">>, _Arg}]}, {[]}, #ctx{negate = false} = Ctx) -> + [#failure{op = keyMapMatch, type = empty_list, ctx = Ctx}]; +match({[{<<"$keyMapMatch">>, _Arg}]}, {[]}, #ctx{negate = true}) -> + []; +match({[{<<"$keyMapMatch">>, Arg}]}, Value, #ctx{negate = true, path = Path} = Ctx) when + is_tuple(Value) +-> + Keys = [Key || {Key, _} <- element(1, Value)], + MatchKey = fun(K) -> match(Arg, K, Ctx#ctx{path = [K | Path]}) end, + lists:flatmap(MatchKey, Keys); +match({[{<<"$keyMapMatch">>, Arg}]}, Value, #ctx{path = Path} = Ctx) when is_tuple(Value) -> + Keys = [Key || {Key, _} <- element(1, Value)], + KeyFailures = [match(Arg, K, Ctx#ctx{path = [K | Path]}) || K <- Keys], + case lists:member([], KeyFailures) of + true -> []; + false -> lists:flatten(KeyFailures) + end; +match({[{<<"$keyMapMatch">>, _Arg}]}, Value, Ctx) -> + [#failure{op = keyMapMatch, type = bad_value, params = [Value], ctx = Ctx}]; % Our comparison operators are fairly straight forward -match({[{<<"$lt">>, Arg}]}, Value, Cmp) -> - Cmp(Value, Arg) < 0; -match({[{<<"$lte">>, Arg}]}, Value, Cmp) -> - Cmp(Value, Arg) =< 0; -match({[{<<"$eq">>, Arg}]}, Value, Cmp) -> - Cmp(Value, Arg) == 0; -match({[{<<"$ne">>, Arg}]}, Value, Cmp) -> - Cmp(Value, Arg) /= 0; -match({[{<<"$gte">>, Arg}]}, Value, Cmp) -> - Cmp(Value, Arg) >= 0; -match({[{<<"$gt">>, Arg}]}, Value, Cmp) -> - Cmp(Value, Arg) > 0; -match({[{<<"$in">>, []}]}, _, _) -> +match({[{<<"$lt">>, Arg}]}, Value, #ctx{cmp = Cmp} = Ctx) -> + compare(lt, Arg, Ctx, Cmp(Value, Arg) < 0); +match({[{<<"$lte">>, Arg}]}, Value, #ctx{cmp = Cmp} = Ctx) -> + compare(lte, Arg, Ctx, Cmp(Value, Arg) =< 0); +match({[{<<"$eq">>, Arg}]}, Value, #ctx{cmp = Cmp} = Ctx) -> + compare(eq, Arg, Ctx, Cmp(Value, Arg) == 0); +match({[{<<"$ne">>, Arg}]}, Value, #ctx{cmp = Cmp} = Ctx) -> + compare(ne, Arg, Ctx, Cmp(Value, Arg) /= 0); +match({[{<<"$gte">>, Arg}]}, Value, #ctx{cmp = Cmp} = Ctx) -> + compare(gte, Arg, Ctx, Cmp(Value, Arg) >= 0); +match({[{<<"$gt">>, Arg}]}, Value, #ctx{cmp = Cmp} = Ctx) -> + compare(gt, Arg, Ctx, Cmp(Value, Arg) > 0); +match({[{<<"$in">>, []}]}, _, #ctx{verbose = false}) -> false; -match({[{<<"$in">>, Args}]}, Values, Cmp) when is_list(Values) -> +match({[{<<"$in">>, Args}]}, Values, #ctx{verbose = false, cmp = Cmp}) when is_list(Values) -> Pred = fun(Arg) -> lists:foldl( fun(Value, Match) -> @@ -498,48 +658,66 @@ match({[{<<"$in">>, Args}]}, Values, Cmp) when is_list(Values) -> ) end, lists:any(Pred, Args); -match({[{<<"$in">>, Args}]}, Value, Cmp) -> +match({[{<<"$in">>, Args}]}, Value, #ctx{verbose = false, cmp = Cmp}) -> Pred = fun(Arg) -> Cmp(Value, Arg) == 0 end, lists:any(Pred, Args); -match({[{<<"$nin">>, []}]}, _, _) -> +match({[{<<"$in">>, Args}]} = Expr, Value, Ctx) -> + match_with_failure(Expr, Value, in, [Args], Ctx); +match({[{<<"$nin">>, []}]}, _, #ctx{verbose = false}) -> true; -match({[{<<"$nin">>, Args}]}, Values, Cmp) when is_list(Values) -> - not match({[{<<"$in">>, Args}]}, Values, Cmp); -match({[{<<"$nin">>, Args}]}, Value, Cmp) -> +match({[{<<"$nin">>, Args}]}, Values, #ctx{verbose = false} = Ctx) when is_list(Values) -> + not match({[{<<"$in">>, Args}]}, Values, Ctx); +match({[{<<"$nin">>, Args}]}, Value, #ctx{verbose = false, cmp = Cmp}) -> Pred = fun(Arg) -> Cmp(Value, Arg) /= 0 end, lists:all(Pred, Args); +match({[{<<"$nin">>, Args}]} = Expr, Value, Ctx) -> + match_with_failure(Expr, Value, nin, [Args], Ctx); % This logic is a bit subtle. Basically, if value is % not undefined, then it exists. -match({[{<<"$exists">>, ShouldExist}]}, Value, _Cmp) -> +match({[{<<"$exists">>, ShouldExist}]}, Value, #ctx{verbose = false}) -> Exists = Value /= undefined, ShouldExist andalso Exists; -match({[{<<"$type">>, Arg}]}, Value, _Cmp) when is_binary(Arg) -> +match({[{<<"$exists">>, ShouldExist}]} = Expr, Value, Ctx) -> + match_with_failure(Expr, Value, exists, [ShouldExist], Ctx); +match({[{<<"$type">>, Arg}]}, Value, #ctx{verbose = false}) when is_binary(Arg) -> Arg == mango_json:type(Value); -match({[{<<"$mod">>, [D, R]}]}, Value, _Cmp) when is_integer(Value) -> +match({[{<<"$type">>, Arg}]} = Expr, Value, Ctx) -> + match_with_failure(Expr, Value, type, [Arg], Ctx); +match({[{<<"$mod">>, [D, R]}]}, Value, #ctx{verbose = false}) when is_integer(Value) -> Value rem D == R; -match({[{<<"$mod">>, _}]}, _Value, _Cmp) -> +match({[{<<"$mod">>, _}]}, _Value, #ctx{verbose = false}) -> false; -match({[{<<"$beginsWith">>, Prefix}]}, Value, _Cmp) when is_binary(Prefix), is_binary(Value) -> +match({[{<<"$mod">>, [D, R]}]} = Expr, Value, Ctx) -> + match_with_failure(Expr, Value, mod, [D, R], Ctx); +match({[{<<"$beginsWith">>, Prefix}]}, Value, #ctx{verbose = false}) when + is_binary(Prefix), is_binary(Value) +-> string:prefix(Value, Prefix) /= nomatch; % When Value is not a string, do not match -match({[{<<"$beginsWith">>, Prefix}]}, _, _Cmp) when is_binary(Prefix) -> +match({[{<<"$beginsWith">>, Prefix}]}, _, #ctx{verbose = false}) when is_binary(Prefix) -> false; -match({[{<<"$regex">>, Regex}]}, Value, _Cmp) when is_binary(Value) -> +match({[{<<"$beginsWith">>, Prefix}]} = Expr, Value, Ctx) -> + match_with_failure(Expr, Value, beginsWith, [Prefix], Ctx); +match({[{<<"$regex">>, Regex}]}, Value, #ctx{verbose = false}) when is_binary(Value) -> try match == re:run(Value, Regex, [{capture, none}]) catch _:_ -> false end; -match({[{<<"$regex">>, _}]}, _Value, _Cmp) -> +match({[{<<"$regex">>, _}]}, _Value, #ctx{verbose = false}) -> false; -match({[{<<"$size">>, Arg}]}, Values, _Cmp) when is_list(Values) -> +match({[{<<"$regex">>, Regex}]} = Expr, Value, Ctx) -> + match_with_failure(Expr, Value, regex, [Regex], Ctx); +match({[{<<"$size">>, Arg}]}, Values, #ctx{verbose = false}) when is_list(Values) -> length(Values) == Arg; -match({[{<<"$size">>, _}]}, _Value, _Cmp) -> +match({[{<<"$size">>, _}]}, _Value, #ctx{verbose = false}) -> false; +match({[{<<"$size">>, Arg}]} = Expr, Value, Ctx) -> + match_with_failure(Expr, Value, size, [Arg], Ctx); % We don't have any choice but to believe that the text % index returned valid matches -match({[{<<"$default">>, _}]}, _Value, _Cmp) -> +match({[{<<"$default">>, _}]}, _Value, #ctx{verbose = false}) -> true; % All other operators are internal assertion errors for % matching because we either should've removed them during @@ -549,22 +727,128 @@ match({[{<<"$", _/binary>> = Op, _}]}, _, _) -> % We need to traverse value to find field. The call to % mango_doc:get_field/2 may return either not_found or % bad_path in which case matching fails. -match({[{Field, Cond}]}, Value, Cmp) -> - case mango_doc:get_field(Value, Field) of - not_found when Cond == {[{<<"$exists">>, false}]} -> - true; - not_found -> - false; - bad_path -> - false; - SubValue when Field == <<"_id">> -> - match(Cond, SubValue, fun mango_json:cmp_raw/2); - SubValue -> - match(Cond, SubValue, Cmp) +match({[{Field, Cond}]}, Value, #ctx{verbose = Verb, path = Path, stack = Stack} = Ctx) -> + InnerCtx = Ctx#ctx{path = [Field | Path]}, + case mango_doc:get_field_with_stack(Value, Field, Stack) of + {not_found, _} when Cond == {[{<<"$exists">>, false}]} -> + case Verb of + true -> []; + false -> true + end; + {not_found, _} -> + case Verb of + true -> [#failure{op = field, type = not_found, ctx = InnerCtx}]; + false -> false + end; + {bad_path, _} -> + case Verb of + true -> [#failure{op = field, type = bad_path, ctx = InnerCtx}]; + false -> false + end; + {SubValue, NewStack} when Field == <<"_id">> -> + match(Cond, SubValue, InnerCtx#ctx{cmp = fun mango_json:cmp_raw/2, stack = NewStack}); + {SubValue, NewStack} -> + match(Cond, SubValue, InnerCtx#ctx{stack = NewStack}) end; -match({[_, _ | _] = _Props} = Sel, _Value, _Cmp) -> +match({[_, _ | _] = _Props} = Sel, _Value, _Ctx) -> error({unnormalized_selector, Sel}). +match_with_failure(Expr, Value, Op, Params, #ctx{negate = Neg} = Ctx) -> + case not match(Expr, Value, Ctx#ctx{verbose = false}) of + Neg -> []; + _ -> [#failure{op = Op, params = Params, ctx = Ctx}] + end. + +compare(_, _, #ctx{verbose = false}, Cond) -> + Cond; +compare(Op, Arg, #ctx{negate = Neg} = Ctx, Cond) -> + case not Cond of + Neg -> []; + _ -> [#failure{op = Op, params = [Arg], ctx = Ctx}] + end. + +format_failure(#failure{op = Op, type = Type, params = Params, ctx = Ctx}) -> + Path = format_path(Ctx#ctx.path), + Msg = format_op(Op, Ctx#ctx.negate, Type, Params), + {[{<<"path">>, Path}, {<<"message">>, list_to_binary(Msg)}]}. + +format_op(Op, _, empty_list, _) -> + io_lib:format("operator $~p was invoked with an empty list", [Op]); +format_op(Op, _, bad_value, [Value]) -> + io_lib:format("operator $~p was invoked with a bad value: ~s", [Op, jiffy:encode(Value)]); +format_op(_, _, not_found, []) -> + io_lib:format("must be present", []); +format_op(_, _, bad_path, []) -> + io_lib:format("used an invalid path", []); +format_op(eq, false, mismatch, [X]) -> + io_lib:format("must be equal to ~s", [jiffy:encode(X)]); +format_op(ne, false, mismatch, [X]) -> + io_lib:format("must not be equal to ~s", [jiffy:encode(X)]); +format_op(lt, false, mismatch, [X]) -> + io_lib:format("must be less than ~s", [jiffy:encode(X)]); +format_op(lte, false, mismatch, [X]) -> + io_lib:format("must be less than or equal to ~s", [jiffy:encode(X)]); +format_op(gt, false, mismatch, [X]) -> + io_lib:format("must be greater than ~s", [jiffy:encode(X)]); +format_op(gte, false, mismatch, [X]) -> + io_lib:format("must be greater than or equal to ~s", [jiffy:encode(X)]); +format_op(in, false, mismatch, [X]) -> + io_lib:format("must be one of ~s", [jiffy:encode(X)]); +format_op(nin, false, mismatch, [X]) -> + io_lib:format("must not be one of ~s", [jiffy:encode(X)]); +format_op(all, false, mismatch, [X]) -> + io_lib:format("must contain all the values in ~s", [jiffy:encode(X)]); +format_op(exists, false, mismatch, [true]) -> + io_lib:format("must be present", []); +format_op(exists, false, mismatch, [false]) -> + io_lib:format("must not be present", []); +format_op(type, false, mismatch, [Type]) -> + io_lib:format("must be of type '~s'", [Type]); +format_op(type, true, mismatch, [Type]) -> + io_lib:format("must not be of type '~s'", [Type]); +format_op(mod, false, mismatch, [D, R]) -> + io_lib:format("must leave a remainder of ~p when divided by ~p", [R, D]); +format_op(mod, true, mismatch, [D, R]) -> + io_lib:format("must leave a remainder other than ~p when divided by ~p", [R, D]); +format_op(regex, false, mismatch, [P]) -> + io_lib:format("must match the pattern '~s'", [P]); +format_op(regex, true, mismatch, [P]) -> + io_lib:format("must not match the pattern '~s'", [P]); +format_op(beginsWith, false, mismatch, [P]) -> + io_lib:format("must begin with '~s'", [P]); +format_op(beginsWith, true, mismatch, [P]) -> + io_lib:format("must not begin with '~s'", [P]); +format_op(size, false, mismatch, [N]) -> + io_lib:format("must contain ~p items", [N]); +format_op(size, true, mismatch, [N]) -> + io_lib:format("must not contain ~p items", [N]); +format_op(eq, true, Type, Params) -> + format_op(ne, false, Type, Params); +format_op(ne, true, Type, Params) -> + format_op(eq, false, Type, Params); +format_op(lt, true, Type, Params) -> + format_op(gte, false, Type, Params); +format_op(lte, true, Type, Params) -> + format_op(gt, false, Type, Params); +format_op(gt, true, Type, Params) -> + format_op(lte, false, Type, Params); +format_op(gte, true, Type, Params) -> + format_op(le, false, Type, Params); +format_op(in, true, Type, Params) -> + format_op(nin, false, Type, Params); +format_op(nin, true, Type, Params) -> + format_op(in, false, Type, Params); +format_op(exists, true, Type, [Exist]) -> + format_op(exists, false, Type, [not Exist]). + +format_path([]) -> + []; +format_path([Item | Rest]) when is_binary(Item) -> + {ok, Path} = mango_util:parse_field(Item), + format_path(Rest) ++ Path; +format_path([Item | Rest]) when is_integer(Item) -> + format_path(Rest) ++ [list_to_binary(integer_to_list(Item))]. + % Returns true if Selector requires all % fields in RequiredFields to exist in any matching documents. @@ -670,6 +954,94 @@ fields({[]}) -> ]} ). +normalize_data_basic_test() -> + Selector = normalize({[{<<"a">>, {[{<<"$data">>, <<"b">>}]}}]}), + ?assertEqual( + {[ + { + <<"a">>, {[{<<"$eq">>, {[{<<"$data">>, [<<"b">>]}]}}]} + } + ]}, + Selector + ). + +normalize_data_path_test() -> + Selector = normalize({[{<<"a">>, {[{<<"$data">>, <<"b.c.42.d">>}]}}]}), + ?assertEqual( + {[ + { + <<"a">>, {[{<<"$eq">>, {[{<<"$data">>, [<<"b">>, <<"c">>, <<"42">>, <<"d">>]}]}}]} + } + ]}, + Selector + ). + +normalize_data_sibling_test() -> + Selector = normalize({[{<<"a">>, {[{<<"$data">>, <<".b">>}]}}]}), + ?assertEqual( + {[ + { + <<"a">>, {[{<<"$eq">>, {[{<<"$data">>, [{[{<<"parent">>, 1}]}, <<"b">>]}]}}]} + } + ]}, + Selector + ). + +normalize_data_parent_test() -> + Selector = normalize({[{<<"a">>, {[{<<"$data">>, <<"..b">>}]}}]}), + ?assertEqual( + {[ + { + <<"a">>, {[{<<"$eq">>, {[{<<"$data">>, [{[{<<"parent">>, 2}]}, <<"b">>]}]}}]} + } + ]}, + Selector + ). + +normalize_data_allowed_operator_test() -> + Selector = normalize({[{<<"a">>, {[{<<"$lt">>, {[{<<"$data">>, <<"b">>}]}}]}}]}), + ?assertEqual( + {[ + { + <<"a">>, {[{<<"$lt">>, {[{<<"$data">>, [<<"b">>]}]}}]} + } + ]}, + Selector + ). + +normalize_data_allowed_operator_all_test() -> + Selector = normalize({[{<<"a">>, {[{<<"$all">>, {[{<<"$data">>, <<"b">>}]}}]}}]}), + ?assertEqual( + {[ + { + <<"a">>, {[{<<"$all">>, {[{<<"$data">>, [<<"b">>]}]}}]} + } + ]}, + Selector + ). + +normalize_data_disallowed_operator_test() -> + Selector = {[{<<"a">>, {[{<<"$allMatch">>, {[{<<"$data">>, <<"b">>}]}}]}}]}, + Error = {mango_error, mango_selector, {bad_arg, '$data', <<"$allMatch">>}}, + ?assertException(throw, Error, normalize(Selector)). + +normalize_data_multi_field_test() -> + Selector = normalize( + {[ + {<<"a">>, {[{<<"$data">>, <<"c">>}]}}, + {<<"b">>, {[{<<"$data">>, <<"c">>}]}} + ]} + ), + ?assertEqual( + {[ + {<<"$and">>, [ + {[{<<"a">>, {[{<<"$eq">>, {[{<<"$data">>, [<<"c">>]}]}}]}}]}, + {[{<<"b">>, {[{<<"$eq">>, {[{<<"$data">>, [<<"c">>]}]}}]}}]} + ]} + ]}, + Selector + ). + is_constant_field_basic_test() -> Selector = normalize({[{<<"A">>, <<"foo">>}]}), Field = <<"A">>, @@ -1070,7 +1442,7 @@ check_beginswith(Field, Prefix) -> % in the middle of test output. match_int(mango_selector:normalize(Selector), ?TEST_DOC). -match_beginswith_test() -> +match_beginswith_errors_test() -> % matching ?assertEqual(true, check_beginswith(<<"_id">>, <<"f">>)), % no match (user_id field in the test doc contains an integer) @@ -1087,4 +1459,910 @@ match_beginswith_test() -> check_beginswith(<<"user_id">>, InvalidArg) ). +check_selector(Selector, Results) -> + SelPos = normalize({[{<<"x">>, Selector}]}), + SelNeg = normalize({[{<<"x">>, {[{<<"$not">>, Selector}]}}]}), + + ListToBool = fun(List) -> + case List of + [] -> true; + [_ | _] -> false + end + end, + + Check = fun({Result, Value}) -> + Doc = {[{<<"x">>, Value}]}, + + ?assertEqual(Result, match_int(SelPos, Doc, false)), + ?assertEqual(Result, ListToBool(match_int(SelPos, Doc, true))), + + ?assertEqual(not Result, match_int(SelNeg, Doc, false)), + ?assertEqual(not Result, ListToBool(match_int(SelNeg, Doc, true))) + end, + + lists:foreach(Check, Results). + +match_lt_test() -> + check_selector({[{<<"$lt">>, 5}]}, [{true, 4}, {false, 5}, {false, 6}]), + + check_selector({[{<<"$lt">>, <<"hello">>}]}, [ + {true, <<"held">>}, + {false, <<"hello">>}, + {false, <<"help">>} + ]), + + check_selector({[{<<"$lt">>, [1, 2, 3]}]}, [ + {true, [1, 2, 2]}, + {true, [1, 2]}, + {false, [1, 2, 3]}, + {false, [1, 2, 4]}, + {false, [1, 3]} + ]). + +match_lte_test() -> + check_selector({[{<<"$lte">>, 5}]}, [{true, 4}, {true, 5}, {false, 6}]), + + check_selector({[{<<"$lte">>, <<"hello">>}]}, [ + {true, <<"held">>}, + {true, <<"hello">>}, + {false, <<"help">>} + ]), + + check_selector({[{<<"$lte">>, [1, 2, 3]}]}, [ + {true, [1, 2, 2]}, + {true, [1, 2]}, + {true, [1, 2, 3]}, + {false, [1, 2, 4]}, + {false, [1, 3]} + ]). + +match_gt_test() -> + check_selector({[{<<"$gt">>, 5}]}, [{false, 4}, {false, 5}, {true, 6}]), + + check_selector({[{<<"$gt">>, <<"hello">>}]}, [ + {false, <<"held">>}, + {false, <<"hello">>}, + {true, <<"help">>} + ]), + + check_selector({[{<<"$gt">>, [1, 2, 3]}]}, [ + {false, [1, 2, 2]}, + {false, [1, 2]}, + {false, [1, 2, 3]}, + {true, [1, 2, 4]}, + {true, [1, 3]} + ]). + +match_gte_test() -> + check_selector({[{<<"$gte">>, 5}]}, [{false, 4}, {true, 5}, {true, 6}]), + + check_selector({[{<<"$gte">>, <<"hello">>}]}, [ + {false, <<"held">>}, + {true, <<"hello">>}, + {true, <<"help">>} + ]), + + check_selector({[{<<"$gte">>, [1, 2, 3]}]}, [ + {false, [1, 2, 2]}, + {false, [1, 2]}, + {true, [1, 2, 3]}, + {true, [1, 2, 4]}, + {true, [1, 3]} + ]). + +match_eq_test() -> + check_selector({[{<<"$eq">>, 5}]}, [{true, 5}, {false, 6}]), + check_selector({[{<<"$eq">>, <<"hello">>}]}, [{true, <<"hello">>}, {false, <<"help">>}]), + + check_selector({[{<<"$eq">>, [1, [2, 3, 4], 5]}]}, [ + {true, [1, [2, 3, 4], 5]}, + {false, [1, [2, 3, 4]]}, + {false, [1, [2, 3, 4], 5, 6]}, + {false, [1, [2, 7, 4], 5]} + ]), + + check_selector({[{<<"$eq">>, {[{<<"a">>, {[{<<"b">>, {[{<<"c">>, 7}]}}]}}]}}]}, [ + {true, {[{<<"a">>, {[{<<"b">>, {[{<<"c">>, 7}]}}]}}]}}, + {false, {[{<<"a">>, {[{<<"b">>, {[{<<"c">>, 8}]}}]}}]}}, + {false, {[{<<"a">>, {[{<<"b">>, {[{<<"d">>, 7}]}}]}}]}}, + {false, {[{<<"a">>, {[{<<"d">>, {[{<<"c">>, 7}]}}]}}]}} + ]). + +match_ne_test() -> + check_selector({[{<<"$ne">>, 5}]}, [{false, 5}, {true, 6}]), + + % the %ne operator still requires a value to be present... + SelInt = normalize({[{<<"x">>, {[{<<"$ne">>, 5}]}}]}), + ?assertEqual(false, match_int(SelInt, {[]})), + + % ... which, due to normalization, means that using $not with $eq does not + % match the empty doc + SelNotEq = normalize({[{<<"$not">>, {[{<<"x">>, 5}]}}]}), + ?assertEqual(false, match_int(SelNotEq, {[]})), + + check_selector({[{<<"$ne">>, <<"hello">>}]}, [{false, <<"hello">>}, {true, <<"help">>}]), + + check_selector({[{<<"$ne">>, [1, [2, 3, 4], 5]}]}, [ + {false, [1, [2, 3, 4], 5]}, + {true, [1, [2, 3, 4]]}, + {true, [1, [2, 3, 4], 5, 6]}, + {true, [1, [2, 7, 4], 5]} + ]), + + check_selector({[{<<"$ne">>, {[{<<"a">>, {[{<<"b">>, {[{<<"c">>, 7}]}}]}}]}}]}, [ + {false, {[{<<"a">>, {[{<<"b">>, {[{<<"c">>, 7}]}}]}}]}}, + {true, {[{<<"a">>, {[{<<"b">>, {[{<<"c">>, 8}]}}]}}]}}, + {true, {[{<<"a">>, {[{<<"b">>, {[{<<"d">>, 7}]}}]}}]}}, + {true, {[{<<"a">>, {[{<<"d">>, {[{<<"c">>, 7}]}}]}}]}} + ]). + +match_in_test() -> + check_selector({[{<<"$in">>, []}]}, [ + {false, 0}, + {false, true}, + {false, <<"foo">>} + ]), + + check_selector( + {[ + {<<"$in">>, [ + 42, + false, + <<"bar">>, + [[<<"nested">>], <<"list">>], + {[{<<"b">>, 2}]} + ]} + ]}, + [ + {true, 42}, + {true, false}, + {true, <<"bar">>}, + {true, {[{<<"b">>, 2}]}}, + + {false, 43}, + {false, true}, + {false, <<"bars">>}, + {false, {[{<<"b">>, 2}, {<<"c">>, 3}]}}, + + % when the input is an array, $in matches if any of the array items + % match... + {true, [0, 42]}, + {true, [0, false]}, + {true, [0, <<"bar">>]}, + {true, [0, {[{<<"b">>, 2}]}]}, + + % ... which means it doesn't directly match when one of the + % candiate values is itself an array + {false, [[<<"nested">>], <<"list">>]}, + {true, [0, [[<<"nested">>], <<"list">>]]} + ] + ). + +match_nin_test() -> + check_selector({[{<<"$nin">>, []}]}, [ + {true, 0}, + {true, true}, + {true, <<"foo">>} + ]), + + check_selector( + {[ + {<<"$nin">>, [ + 42, + false, + <<"bar">>, + [[<<"nested">>], <<"list">>], + {[{<<"b">>, 2}]} + ]} + ]}, + [ + {false, 42}, + {false, false}, + {false, <<"bar">>}, + {false, {[{<<"b">>, 2}]}}, + + {true, 43}, + {true, true}, + {true, <<"bars">>}, + {true, {[{<<"b">>, 2}, {<<"c">>, 3}]}}, + + % when the input is an array, $nin matches if none of the array items + % match... + {false, [0, 42]}, + {false, [0, false]}, + {false, [0, <<"bar">>]}, + {false, [0, {[{<<"b">>, 2}]}]}, + + % ... which means it doesn't directly match when one of the + % candiate values is itself an array + {true, [[<<"nested">>], <<"list">>]}, + {false, [0, [[<<"nested">>], <<"list">>]]} + ] + ). + +match_all_test() -> + % { "$all": [] } matches nothing, not even arrays + check_selector({[{<<"$all">>, []}]}, [ + {false, []}, + {false, [42]}, + {false, {[]}}, + {false, <<"foo">>} + ]), + + % normally, input lists can contain the required items in any order + check_selector({[{<<"$all">>, [1, 2, 3, 4]}]}, [ + {true, [3, 2, 4, 1]}, + {true, [0, 4, 3, 5, 2, 1, 6]}, + {false, [3, 2, 4]}, + {false, []} + ]), + + % negation means the input must lack at least one of the items + check_selector({[{<<"$not">>, {[{<<"$all">>, [1, 2, 3, 4]}]}}]}, [ + {true, [2, 4, 1]}, + {false, [2, 4, 1, 3]}, + {true, []} + ]), + + % the special $all: [List] form allows the input to exactly match List... + check_selector({[{<<"$all">>, [[1, 2, 3, 4]]}]}, [ + {true, [1, 2, 3, 4]}, + {false, [4, 3, 2, 1]}, + {false, [1, 3, 4]}, + {false, []}, + % ... or to contain List + {true, [5, [1, 2, 3, 4], 6]}, + {false, [5, [1, 3, 4], 6]}, + {false, [5, [1, 3, 2, 4], 6]} + ]), + + % the special behaviour of $all: [X] only applies when X is a list + check_selector({[{<<"$all">>, [<<"hello">>]}]}, [ + {false, <<"hello">>}, + {true, [<<"hello">>]}, + {true, [0, <<"hello">>, 1]}, + {false, []} + ]), + + % values must match exactly and not contain extra fields + check_selector({[{<<"$all">>, [{[{<<"a">>, 1}]}]}]}, [ + {true, [{[{<<"a">>, 1}]}]}, + {false, [{[{<<"a">>, 1}, {<<"b">>, 2}]}]} + ]). + +match_exists_test() -> + check_selector({[{<<"x">>, {[{<<"$exists">>, true}]}}]}, [ + {true, {[{<<"x">>, 0}]}}, + {false, {[{<<"y">>, 0}]}}, + {false, {[]}} + ]), + + check_selector({[{<<"x">>, {[{<<"$exists">>, false}]}}]}, [ + {false, {[{<<"x">>, 0}]}}, + {true, {[{<<"y">>, 0}]}}, + {true, {[]}} + ]), + + % due to normalizing to { "x": { "$ne": 0 } }, this does not match the empty doc + SelNeg = normalize({[{<<"x">>, {[{<<"$not">>, {[{<<"$eq">>, 0}]}}]}}]}), + SelPos = normalize({[{<<"x">>, 0}]}), + ?assertEqual(false, match_int(SelNeg, {[]})), + ?assertEqual(false, match_int(SelPos, {[]})), + + % including { "$exists": true } in the negated part *does* match the empty doc + check_selector( + {[ + {<<"x">>, + {[ + {<<"$not">>, + {[ + {<<"$exists">>, true}, + {<<"$eq">>, 0} + ]}} + ]}} + ]}, + [ + {true, {[{<<"x">>, 1}]}}, + {false, {[{<<"x">>, 0}]}}, + {true, {[]}} + ] + ). + +match_type_test() -> + check_selector({[{<<"$type">>, <<"null">>}]}, [ + {true, null}, + {false, false}, + {false, {[]}} + ]), + + check_selector({[{<<"$type">>, <<"boolean">>}]}, [ + {true, true}, + {true, false}, + {false, 0} + ]), + + check_selector({[{<<"$type">>, <<"number">>}]}, [ + {true, 42}, + {true, 3.14}, + {true, 0}, + {false, true}, + {false, [1]}, + {false, <<"1">>} + ]), + + check_selector({[{<<"$type">>, <<"string">>}]}, [ + {true, <<"">>}, + {true, <<"hello">>}, + {false, []} + ]), + + check_selector({[{<<"$type">>, <<"array">>}]}, [ + {true, []}, + {true, [1, 2]}, + {false, {[]}}, + {false, <<"hi">>} + ]), + + check_selector({[{<<"$type">>, <<"object">>}]}, [ + {true, {[]}}, + {true, {[{<<"a">>, 1}]}}, + {false, [{<<"a">>, 1}]}, + {false, null} + ]). + +match_regex_test() -> + check_selector({[{<<"$regex">>, <<"^[0-9a-f]+$">>}]}, [ + {false, <<"">>}, + {true, <<"3a0df5e">>}, + {false, <<"3a0gf5e">>}, + {false, 42} + ]). + +match_beginswith_test() -> + check_selector({[{<<"$beginsWith">>, <<"foo">>}]}, [ + {true, <<"foo">>}, + {true, <<"food">>}, + {true, <<"fool me once">>}, + {false, <<"more food">>}, + {false, <<"fo">>}, + {false, 42} + ]). + +match_mod_test() -> + check_selector({[{<<"$mod">>, [28, 1]}]}, [ + {true, 1}, + {true, 29}, + {true, 57}, + {false, 58}, + {false, <<"57">>} + ]). + +match_size_test() -> + check_selector({[{<<"$size">>, 3}]}, [ + {false, 3}, + {false, <<"fun">>}, + {true, [0, 0, 0]}, + {false, [0, 0]}, + {false, [0, 0, 0, 0]} + ]). + +match_allmatch_test() -> + % $allMatch is defined to return false for empty lists + check_selector({[{<<"$allMatch">>, {[{<<"$eq">>, 0}]}}]}, [ + {false, []}, + {true, [0]}, + {false, [1]}, + {false, [0, 1]} + ]), + + % because of their behaviour on empty lists, { "$not": { "$allMatch": S } } + % is not equivalent to { "$elemMatch": { "$not": S } } + check_selector({[{<<"$elemMatch">>, {[{<<"$ne">>, 0}]}}]}, [ + {false, []}, + {false, [0]}, + {true, [1]}, + {true, [0, 1]} + ]). + +match_elemmatch_test() -> + check_selector({[{<<"$elemMatch">>, {[{<<"$eq">>, 0}]}}]}, [ + {false, []}, + {true, [0]}, + {false, [1]}, + {true, [0, 1]} + ]). + +match_keymapmatch_test() -> + check_selector({[{<<"$keyMapMatch">>, {[{<<"$regex">>, <<"^[a-z]+$">>}]}}]}, [ + {true, {[{<<"hello">>, 0}]}}, + {true, {[{<<"a">>, 1}, {<<"b">>, 2}]}}, + {true, {[{<<"a">>, 1}, {<<"b4">>, 2}]}}, + {false, {[{<<"b4">>, 2}]}}, + {false, {[]}} + ]). + +match_object_test() -> + Doc1 = {[]}, + Doc2 = {[{<<"x">>, {[]}}]}, + Doc3 = {[{<<"x">>, {[{<<"a">>, 1}]}}]}, + Doc4 = {[{<<"x">>, {[{<<"a">>, 1}, {<<"b">>, 2}]}}]}, + Doc5 = {[{<<"x">>, []}]}, + + % the empty selector matches any document + SelEmpty = normalize({[]}), + ?assertEqual({[]}, SelEmpty), + ?assertEqual(true, match_int(SelEmpty, Doc1)), + ?assertEqual(true, match_int(SelEmpty, Doc2)), + ?assertEqual(true, match_int(SelEmpty, Doc3)), + ?assertEqual(true, match_int(SelEmpty, Doc4)), + ?assertEqual(true, match_int(SelEmpty, Doc5)), + + % an inner empty object selector matches only empty objects + SelEmptyField = normalize({[{<<"x">>, {[]}}]}), + ?assertEqual({[{<<"x">>, {[{<<"$eq">>, {[]}}]}}]}, SelEmptyField), + ?assertEqual(false, match_int(SelEmptyField, Doc1)), + ?assertEqual(true, match_int(SelEmptyField, Doc2)), + ?assertEqual(false, match_int(SelEmptyField, Doc3)), + ?assertEqual(false, match_int(SelEmptyField, Doc4)), + ?assertEqual(false, match_int(SelEmptyField, Doc5)), + + % negated empty object selector matches a value which is present and is not the empty object + SelNotEmptyField = normalize({[{<<"$not">>, {[{<<"x">>, {[]}}]}}]}), + ?assertEqual({[{<<"x">>, {[{<<"$ne">>, {[]}}]}}]}, SelNotEmptyField), + ?assertEqual(false, match_int(SelNotEmptyField, Doc1)), + ?assertEqual(false, match_int(SelNotEmptyField, Doc2)), + ?assertEqual(true, match_int(SelNotEmptyField, Doc3)), + ?assertEqual(true, match_int(SelNotEmptyField, Doc4)), + ?assertEqual(true, match_int(SelNotEmptyField, Doc5)), + + % inner object selectors with fields match objects with at least those fields + Sel1Field = normalize({[{<<"x">>, {[{<<"a">>, 1}]}}]}), + ?assertEqual({[{<<"x.a">>, {[{<<"$eq">>, 1}]}}]}, Sel1Field), + ?assertEqual(false, match_int(Sel1Field, Doc1)), + ?assertEqual(false, match_int(Sel1Field, Doc2)), + ?assertEqual(true, match_int(Sel1Field, Doc3)), + ?assertEqual(true, match_int(Sel1Field, Doc4)), + ?assertEqual(false, match_int(Sel1Field, Doc5)), + + % inner object selectors with multiple fields are normalized with $and + Sel2Field = normalize({[{<<"x">>, {[{<<"a">>, 1}, {<<"b">>, 2}]}}]}), + ?assertEqual( + {[ + {<<"$and">>, [ + {[{<<"x.a">>, {[{<<"$eq">>, 1}]}}]}, + {[{<<"x.b">>, {[{<<"$eq">>, 2}]}}]} + ]} + ]}, + Sel2Field + ), + ?assertEqual(false, match_int(Sel2Field, Doc1)), + ?assertEqual(false, match_int(Sel2Field, Doc2)), + ?assertEqual(false, match_int(Sel2Field, Doc3)), + ?assertEqual(true, match_int(Sel2Field, Doc4)), + ?assertEqual(false, match_int(Sel2Field, Doc5)), + + % check shorthand syntax + SelShort = normalize({[{<<"x.b">>, 2}]}), + ?assertEqual({[{<<"x.b">>, {[{<<"$eq">>, 2}]}}]}, SelShort), + ?assertEqual(false, match_int(SelShort, Doc1)), + ?assertEqual(false, match_int(SelShort, Doc2)), + ?assertEqual(false, match_int(SelShort, Doc3)), + ?assertEqual(true, match_int(SelShort, Doc4)), + ?assertEqual(false, match_int(SelShort, Doc5)). + +match_data_test() -> + SelAbs = normalize({[{<<"a">>, {[{<<"$gt">>, {[{<<"$data">>, <<"b">>}]}}]}}]}), + ?assertEqual(true, match_int(SelAbs, {[{<<"a">>, 2}, {<<"b">>, 1}]})), + ?assertEqual(false, match_int(SelAbs, {[{<<"a">>, 2}, {<<"b">>, 2}]})), + + ?assertEqual(false, match_int(SelAbs, {[{<<"a">>, 2}]})), + ?assertMatch( + [#failure{op = data, type = not_found, params = [], ctx = #ctx{path = [<<"b">>]}}], + match_int(SelAbs, {[{<<"a">>, 2}]}, true) + ), + + ?assertEqual(false, match_int(SelAbs, {[{<<"b">>, 2}]})), + ?assertMatch( + [#failure{op = field, type = not_found, params = [], ctx = #ctx{path = [<<"a">>]}}], + match_int(SelAbs, {[{<<"b">>, 2}]}, true) + ), + + SelRel = normalize({[{<<"a.b">>, {[{<<"$gt">>, {[{<<"$data">>, <<".c">>}]}}]}}]}), + ?assertEqual(true, match_int(SelRel, {[{<<"a">>, {[{<<"b">>, 2}, {<<"c">>, 1}]}}]})), + ?assertEqual(false, match_int(SelRel, {[{<<"a">>, {[{<<"b">>, 2}, {<<"c">>, 2}]}}]})), + + SelRelOutOfBounds = normalize({[{<<"a">>, {[{<<"$gt">>, {[{<<"$data">>, <<"..b">>}]}}]}}]}), + ?assertEqual(false, match_int(SelRelOutOfBounds, {[{<<"a">>, 1}]})), + + SelRelAllMatch = normalize( + {[ + {<<"a">>, + {[ + {<<"$allMatch">>, + {[ + {<<"b">>, {[{<<"$gt">>, {[{<<"$data">>, <<".c">>}]}}]}} + ]}} + ]}} + ]} + ), + ?assertEqual( + true, + match_int( + SelRelAllMatch, + {[ + {<<"a">>, [ + {[{<<"b">>, 2}, {<<"c">>, 1}]}, + {[{<<"b">>, 5}, {<<"c">>, 3}]}, + {[{<<"b">>, 7}, {<<"c">>, 6}]} + ]} + ]} + ) + ), + ?assertEqual( + false, + match_int( + SelRelAllMatch, + {[ + {<<"a">>, [ + {[{<<"b">>, 2}, {<<"c">>, 1}]}, + {[{<<"b">>, 2}, {<<"c">>, 3}]}, + {[{<<"b">>, 7}, {<<"c">>, 6}]} + ]} + ]} + ) + ). + +match_and_test() -> + % $and with an empty array matches anything + SelEmpty = normalize({[{<<"x">>, {[{<<"$and">>, []}]}}]}), + ?assertEqual(true, match_int(SelEmpty, {[{<<"x">>, 0}]})), + ?assertEqual(true, match_int(SelEmpty, {[{<<"x">>, false}]})), + ?assertEqual(true, match_int(SelEmpty, {[{<<"x">>, []}]})), + ?assertEqual(true, match_int(SelEmpty, {[]})), + + % due to { "$or": [] } matching anything, negating { "$and": [] } also + % matches anything + SelNotEmpty = normalize({[{<<"x">>, {[{<<"$not">>, {[{<<"$and">>, []}]}}]}}]}), + ?assertEqual(true, match_int(SelNotEmpty, {[{<<"x">>, 0}]})), + ?assertEqual(true, match_int(SelNotEmpty, {[{<<"x">>, false}]})), + ?assertEqual(true, match_int(SelNotEmpty, {[{<<"x">>, []}]})), + + % and, because { "x": { "$and": [A, B] } } normalizes to + % { "$and": [{ "x": A }, { "x": B }] }, that means + % { "x": { "$not": { "$and": [] } } } normalizes to { "$or": [] }, + % so it matches docs where "x" is not present + ?assertEqual(true, match_int(SelNotEmpty, {[]})), + + % $and with multiple selectors matches if all selectors match + SelMulti = normalize( + {[ + {<<"x">>, + {[ + {<<"$and">>, [ + {[{<<"$gt">>, 3}]}, + {[{<<"$lt">>, 7}]} + ]} + ]}} + ]} + ), + ?assertEqual(true, match_int(SelMulti, {[{<<"x">>, 6}]})), + ?assertEqual(false, match_int(SelMulti, {[{<<"x">>, 2}]})), + ?assertEqual(false, match_int(SelMulti, {[{<<"x">>, 9}]})), + ?assertEqual(false, match_int(SelMulti, {[]})), + + % $not -> $and with multiple selectors matches if any selector does not match + SelNotMulti = normalize( + {[ + {<<"x">>, + {[ + {<<"$not">>, + {[ + {<<"$and">>, [ + {[{<<"$gt">>, 3}]}, + {[{<<"$lt">>, 7}]} + ]} + ]}} + ]}} + ]} + ), + ?assertEqual(false, match_int(SelNotMulti, {[{<<"x">>, 6}]})), + ?assertEqual(true, match_int(SelNotMulti, {[{<<"x">>, 2}]})), + ?assertEqual(true, match_int(SelNotMulti, {[{<<"x">>, 9}]})), + ?assertEqual(false, match_int(SelNotMulti, {[]})). + +match_or_test() -> + % $or with an empty array matches anything + SelEmpty = normalize({[{<<"x">>, {[{<<"$or">>, []}]}}]}), + ?assertEqual(true, match_int(SelEmpty, {[{<<"x">>, 0}]})), + ?assertEqual(true, match_int(SelEmpty, {[{<<"x">>, false}]})), + ?assertEqual(true, match_int(SelEmpty, {[{<<"x">>, []}]})), + ?assertEqual(true, match_int(SelEmpty, {[]})), + + % similar to $and, due to { "$or": [] } matching anything and our + % normalization rules, negating $or also matches anything + SelNotEmpty = normalize({[{<<"x">>, {[{<<"$not">>, {[{<<"$or">>, []}]}}]}}]}), + ?assertEqual(true, match_int(SelNotEmpty, {[{<<"x">>, 0}]})), + ?assertEqual(true, match_int(SelNotEmpty, {[{<<"x">>, false}]})), + ?assertEqual(true, match_int(SelNotEmpty, {[{<<"x">>, []}]})), + ?assertEqual(true, match_int(SelNotEmpty, {[]})), + + % $or with multiple selectors matches if any selector matches + SelMulti = normalize( + {[ + {<<"x">>, + {[ + {<<"$or">>, [ + {[{<<"$lt">>, 3}]}, + {[{<<"$gt">>, 7}]} + ]} + ]}} + ]} + ), + ?assertEqual(false, match_int(SelMulti, {[{<<"x">>, 6}]})), + ?assertEqual(true, match_int(SelMulti, {[{<<"x">>, 2}]})), + ?assertEqual(true, match_int(SelMulti, {[{<<"x">>, 9}]})), + ?assertEqual(false, match_int(SelMulti, {[]})), + + % $not -> $or with multiple selectors matches if no selector matches + SelNotMulti = normalize( + {[ + {<<"x">>, + {[ + {<<"$not">>, + {[ + {<<"$or">>, [ + {[{<<"$lt">>, 3}]}, + {[{<<"$gt">>, 7}]} + ]} + ]}} + ]}} + ]} + ), + ?assertEqual(true, match_int(SelNotMulti, {[{<<"x">>, 6}]})), + ?assertEqual(false, match_int(SelNotMulti, {[{<<"x">>, 2}]})), + ?assertEqual(false, match_int(SelNotMulti, {[{<<"x">>, 9}]})), + ?assertEqual(false, match_int(SelNotMulti, {[]})). + +match_nor_test() -> + % $nor with an empty array matches anything + SelEmpty = normalize({[{<<"x">>, {[{<<"$nor">>, []}]}}]}), + ?assertEqual(true, match_int(SelEmpty, {[{<<"x">>, 0}]})), + ?assertEqual(true, match_int(SelEmpty, {[{<<"x">>, false}]})), + ?assertEqual(true, match_int(SelEmpty, {[{<<"x">>, []}]})), + ?assertEqual(true, match_int(SelEmpty, {[]})), + + % $nor with multiple selectors matches if no selector matches + SelMulti = normalize( + {[ + {<<"x">>, + {[ + {<<"$nor">>, [ + {[{<<"$lt">>, 3}]}, + {[{<<"$gt">>, 7}]} + ]} + ]}} + ]} + ), + ?assertEqual(true, match_int(SelMulti, {[{<<"x">>, 6}]})), + ?assertEqual(false, match_int(SelMulti, {[{<<"x">>, 2}]})), + ?assertEqual(false, match_int(SelMulti, {[{<<"x">>, 9}]})), + ?assertEqual(false, match_int(SelMulti, {[]})). + +match_failures_object_test() -> + Selector = normalize( + {[ + {<<"a">>, 1}, + {<<"b">>, {[{<<"c">>, 3}]}} + ]} + ), + + Fails0 = match_int( + Selector, + {[ + {<<"a">>, 1}, + {<<"b">>, {[{<<"c">>, 3}]}} + ]}, + true + ), + ?assertEqual([], Fails0), + + Fails1 = match_int( + Selector, + {[ + {<<"a">>, 0}, + {<<"b">>, {[{<<"c">>, 3}]}} + ]}, + true + ), + ?assertMatch( + [#failure{op = eq, type = mismatch, params = [1], ctx = #ctx{path = [<<"a">>]}}], + Fails1 + ), + + Fails2 = match_int( + Selector, + {[ + {<<"a">>, 1}, + {<<"b">>, {[{<<"c">>, 4}]}} + ]}, + true + ), + ?assertMatch( + [#failure{op = eq, type = mismatch, params = [3], ctx = #ctx{path = [<<"b.c">>]}}], + Fails2 + ). + +match_failures_elemmatch_test() -> + SelElemMatch = normalize( + {[ + {<<"a">>, + {[ + {<<"$elemMatch">>, {[{<<"$gt">>, 4}]}} + ]}} + ]} + ), + + Fails0 = match_int( + SelElemMatch, {[{<<"a">>, [5, 3, 2]}]}, true + ), + ?assertEqual([], Fails0), + + Fails1 = match_int( + SelElemMatch, {[{<<"a">>, []}]}, true + ), + ?assertMatch( + [#failure{op = elemMatch, type = empty_list, params = [], ctx = #ctx{path = [<<"a">>]}}], + Fails1 + ), + + Fails2 = match_int( + SelElemMatch, {[{<<"a">>, [3, 2]}]}, true + ), + ?assertMatch( + [ + #failure{op = gt, type = mismatch, params = [4], ctx = #ctx{path = [0, <<"a">>]}}, + #failure{op = gt, type = mismatch, params = [4], ctx = #ctx{path = [1, <<"a">>]}} + ], + Fails2 + ). + +match_failures_allmatch_test() -> + SelAllMatch = normalize( + {[ + {<<"a">>, + {[ + {<<"$allMatch">>, {[{<<"$gt">>, 4}]}} + ]}} + ]} + ), + + Fails0 = match_int( + SelAllMatch, {[{<<"a">>, [5]}]}, true + ), + ?assertEqual([], Fails0), + + Fails1 = match_int( + SelAllMatch, {[{<<"a">>, [4]}]}, true + ), + ?assertMatch( + [#failure{op = gt, type = mismatch, params = [4], ctx = #ctx{path = [0, <<"a">>]}}], + Fails1 + ), + + Fails2 = match_int( + SelAllMatch, {[{<<"a">>, [5, 6, 3, 7, 0]}]}, true + ), + ?assertMatch( + [ + #failure{op = gt, type = mismatch, params = [4], ctx = #ctx{path = [2, <<"a">>]}}, + #failure{op = gt, type = mismatch, params = [4], ctx = #ctx{path = [4, <<"a">>]}} + ], + Fails2 + ). + +match_failures_allmatch_object_test() -> + SelAllMatch = normalize( + {[ + {<<"a.b">>, + {[ + {<<"$allMatch">>, {[{<<"c">>, {[{<<"$gt">>, 4}]}}]}} + ]}} + ]} + ), + + Fails0 = match_int( + SelAllMatch, {[{<<"a">>, {[{<<"b">>, [{[{<<"c">>, 5}]}]}]}}]}, true + ), + ?assertEqual([], Fails0), + + Fails1 = match_int( + SelAllMatch, {[{<<"a">>, {[{<<"b">>, [{[{<<"c">>, 4}]}]}]}}]}, true + ), + ?assertMatch( + [ + #failure{ + op = gt, type = mismatch, params = [4], ctx = #ctx{path = [<<"c">>, 0, <<"a.b">>]} + } + ], + Fails1 + ), + + Fails2 = match_int( + SelAllMatch, + {[{<<"a">>, {[{<<"b">>, [{[{<<"c">>, 5}]}, {[{<<"c">>, 6}]}, {[{<<"c">>, 3}]}]}]}}]}, + true + ), + ?assertMatch( + [ + #failure{ + op = gt, type = mismatch, params = [4], ctx = #ctx{path = [<<"c">>, 2, <<"a.b">>]} + } + ], + Fails2 + ), + + Fails3 = match_int( + SelAllMatch, + {[{<<"a">>, {[{<<"b">>, [{[{<<"c">>, 1}]}, {[]}]}]}}]}, + true + ), + ?assertMatch( + [ + #failure{ + op = gt, type = mismatch, params = [4], ctx = #ctx{path = [<<"c">>, 0, <<"a.b">>]} + }, + #failure{ + op = field, + type = not_found, + params = [], + ctx = #ctx{path = [<<"c">>, 1, <<"a.b">>]} + } + ], + Fails3 + ). + +format_path_test() -> + ?assertEqual([], format_path([])), + ?assertEqual([<<"a">>], format_path([<<"a">>])), + ?assertEqual([<<"a">>, <<"b">>], format_path([<<"b">>, <<"a">>])), + ?assertEqual([<<"a">>, <<"b">>, <<"c">>], format_path([<<"b.c">>, <<"a">>])), + ?assertEqual([<<"a">>, <<"42">>, <<"b">>, <<"c">>], format_path([<<"b.c">>, 42, <<"a">>])). + +bench(Name, Selector, Doc) -> + Sel1 = normalize(Selector), + [Normal, Verbose] = erlperf:compare( + [ + #{runner => fun() -> match_int(Sel1, Doc, V) end} + || V <- [false, true] + ], + #{} + ), + ?debugFmt("~nbench[~s: normal ] = ~p~n", [Name, Normal]), + ?debugFmt("~nbench[~s: verbose] = ~p~n", [Name, Verbose]). + +bench_and_test() -> + Sel = + {[ + {<<"x">>, + {[ + {<<"$and">>, [{[{<<"$gt">>, V}]} || V <- [100, 200, 300, 400, 500]]} + ]}} + ]}, + Doc = {[{<<"x">>, 25}]}, + bench("$and", Sel, Doc). + +bench_allmatch_test() -> + Sel = + {[ + {<<"x">>, + {[ + {<<"$allMatch">>, {[{<<"$gt">>, 10}]}} + ]}} + ]}, + Doc = + {[ + {<<"x">>, [0, 23, 45, 67, 89, 12, 34, 56, 78]} + ]}, + bench("$allMatch", Sel, Doc). + -endif. diff --git a/src/mango/src/mango_util.erl b/src/mango/src/mango_util.erl index 837cbf3dbe8..5bd8dc15cc8 100644 --- a/src/mango/src/mango_util.erl +++ b/src/mango/src/mango_util.erl @@ -41,6 +41,7 @@ join/2, parse_field/1, + parse_field/2, cached_re/2 ]). @@ -350,16 +351,36 @@ cached_re(Name, RE) -> end. parse_field(Field) -> - case binary:match(Field, <<"\\">>, []) of - nomatch -> - % Fast path, no regex required - {ok, check_non_empty(Field, binary:split(Field, <<".">>, [global]))}; - _ -> - parse_field_slow(Field) - end. + parse_field(Field, absolute). + +parse_field(Field, Mode) -> + Parts = + case binary:match(Field, <<"\\">>, []) of + nomatch -> + % Fast path, no regex required + binary:split(Field, <<".">>, [global]); + _ -> + parse_field_slow(Field) + end, + {Prefix, Path} = + case Mode of + absolute -> + {[], Parts}; + relative -> + lists:foldl( + fun + (<<>>, {[], []}) -> {[{[{<<"parent">>, 1}]}], []}; + (<<>>, {[{[{<<"parent">>, N}]}], []}) -> {[{[{<<"parent">>, N + 1}]}], []}; + (Pt, {Pre, Pts}) -> {Pre, Pts ++ [Pt]} + end, + {[], []}, + Parts + ) + end, + {ok, check_non_empty(Field, Prefix ++ Path)}. parse_field_slow(Field) -> - Path = lists:map( + lists:map( fun (P) when P =:= <<>> -> ?MANGO_ERROR({invalid_field_name, Field}); @@ -367,8 +388,7 @@ parse_field_slow(Field) -> re:replace(P, <<"\\\\">>, <<>>, [global, {return, binary}]) end, re:split(Field, <<"(?>) - ), - {ok, Path}. + ). check_non_empty(Field, Parts) -> case lists:member(<<>>, Parts) of diff --git a/test/elixir/test/config/suite.elixir b/test/elixir/test/config/suite.elixir index 81ed1e63d7e..e5046b6597d 100644 --- a/test/elixir/test/config/suite.elixir +++ b/test/elixir/test/config/suite.elixir @@ -519,6 +519,19 @@ "serial execution is not spuriously counted as loop on test_rewrite_suite_db", "serial execution is not spuriously counted as loop on test_rewrite_suite_db%2Fwith_slashes" ], + "ValidateDocUpdateTest": [ + "JavaScript VDU accepts a valid document", + "JavaScript VDU rejects an invalid document", + "JavaScript VDU accepts a valid change", + "JavaScript VDU rejects an invalid change", + "Mango VDU accepts a valid document", + "Mango VDU rejects an invalid document", + "updating a Mango VDU updates its effects", + "converting a Mango VDU to JavaScript updates its effects", + "deleting a Mango VDU removes its effects", + "Mango VDU rejects a doc if any existing ddoc fails to match", + "Mango VDU allows comparisons via $data", + ], "SecurityValidationTest": [ "Author presence and user security", "Author presence and user security when replicated", diff --git a/test/elixir/test/validate_doc_update_test.exs b/test/elixir/test/validate_doc_update_test.exs new file mode 100644 index 00000000000..97f7ad82bd6 --- /dev/null +++ b/test/elixir/test/validate_doc_update_test.exs @@ -0,0 +1,256 @@ +defmodule ValidateDocUpdateTest do + use CouchTestCase + + @moduledoc """ + Test validate_doc_update behaviour + """ + + @js_type_check %{ + language: "javascript", + + validate_doc_update: ~s""" + function (newDoc) { + if (!newDoc.type) { + throw {forbidden: 'Documents must have a type field'}; + } + } + """ + } + + @tag :with_db + test "JavaScript VDU accepts a valid document", context do + db = context[:db_name] + Couch.put("/#{db}/_design/js-test", body: @js_type_check) + + resp = Couch.put("/#{db}/doc", body: %{"type" => "movie"}) + assert resp.status_code == 201 + assert resp.body["ok"] == true + end + + @tag :with_db + test "JavaScript VDU rejects an invalid document", context do + db = context[:db_name] + Couch.put("/#{db}/_design/js-test", body: @js_type_check) + + resp = Couch.put("/#{db}/doc", body: %{"not" => "valid"}) + assert resp.status_code == 403 + assert resp.body["error"] == "forbidden" + end + + @js_change_check %{ + language: "javascript", + + validate_doc_update: ~s""" + function (newDoc, oldDoc) { + if (oldDoc && newDoc.type !== oldDoc.type) { + throw {forbidden: 'Documents cannot change their type field'}; + } + } + """ + } + + @tag :with_db + test "JavaScript VDU accepts a valid change", context do + db = context[:db_name] + Couch.put("/#{db}/_design/js-test", body: @js_change_check) + + Couch.put("/#{db}/doc", body: %{"type" => "movie"}) + + doc = Couch.get("/#{db}/doc").body + updated = doc |> Map.merge(%{"type" => "movie", "title" => "Duck Soup"}) + resp = Couch.put("/#{db}/doc", body: updated) + + assert resp.status_code == 201 + end + + @tag :with_db + test "JavaScript VDU rejects an invalid change", context do + db = context[:db_name] + Couch.put("/#{db}/_design/js-test", body: @js_change_check) + + Couch.put("/#{db}/doc", body: %{"type" => "movie"}) + + doc = Couch.get("/#{db}/doc").body + updated = doc |> Map.put("type", "director") + resp = Couch.put("/#{db}/doc", body: updated) + + assert resp.status_code == 403 + end + + @mango_type_check %{ + language: "query", + + validate_doc_update: %{ + "newDoc" => %{"type" => %{"$exists" => true}} + } + } + + @tag :with_db + test "Mango VDU accepts a valid document", context do + db = context[:db_name] + resp = Couch.put("/#{db}/_design/mango-test", body: @mango_type_check) + assert resp.status_code == 201 + + resp = Couch.put("/#{db}/doc", body: %{"type" => "movie"}) + assert resp.status_code == 201 + assert resp.body["ok"] == true + end + + @tag :with_db + test "Mango VDU rejects an invalid document", context do + db = context[:db_name] + resp = Couch.put("/#{db}/_design/mango-test", body: @mango_type_check) + assert resp.status_code == 201 + + resp = Couch.put("/#{db}/doc", body: %{"no" => "type"}) + assert resp.status_code == 403 + assert resp.body["error"] == "forbidden" + assert resp.body["reason"] == [ + %{"path" => ["newDoc", "type"], "message" => "must be present"} + ] + end + + @tag :with_db + test "updating a Mango VDU updates its effects", context do + db = context[:db_name] + + resp = Couch.put("/#{db}/_design/mango-test", body: @mango_type_check) + assert resp.status_code == 201 + + ddoc = %{ + language: "query", + + validate_doc_update: %{ + "newDoc" => %{ + "type" => %{"$type" => "string"}, + "year" => %{"$lt" => 2026} + } + } + } + resp = Couch.put("/#{db}/_design/mango-test", body: ddoc, query: %{rev: resp.body["rev"]}) + assert resp.status_code == 201 + + resp = Couch.put("/#{db}/doc1", body: %{"type" => "movie", "year" => 1994}) + assert resp.status_code == 201 + + resp = Couch.put("/#{db}/doc2", body: %{"type" => 42, "year" => 1994}) + assert resp.status_code == 403 + assert resp.body["error"] == "forbidden" + + resp = Couch.put("/#{db}/doc3", body: %{"type" => "movie", "year" => 2094}) + assert resp.status_code == 403 + assert resp.body["error"] == "forbidden" + end + + @tag :with_db + test "converting a Mango VDU to JavaScript updates its effects", context do + db = context[:db_name] + + resp = Couch.put("/#{db}/_design/mango-test", body: @mango_type_check) + assert resp.status_code == 201 + + ddoc = %{ + language: "javascript", + + validate_doc_update: ~s""" + function (newDoc) { + if (typeof newDoc.year !== 'number') { + throw {forbidden: 'Documents must have a valid year field'}; + } + } + """ + } + resp = Couch.put("/#{db}/_design/mango-test", body: ddoc, query: %{rev: resp.body["rev"]}) + assert resp.status_code == 201 + + resp = Couch.put("/#{db}/doc1", body: %{"year" => 1994}) + assert resp.status_code == 201 + + resp = Couch.put("/#{db}/doc2", body: %{"year" => "1994"}) + assert resp.status_code == 403 + assert resp.body["error"] == "forbidden" + end + + @tag :with_db + test "deleting a Mango VDU removes its effects", context do + db = context[:db_name] + + resp = Couch.put("/#{db}/_design/mango-test", body: @mango_type_check) + assert resp.status_code == 201 + + resp = Couch.delete("/#{db}/_design/mango-test", query: %{rev: resp.body["rev"]}) + assert resp.status_code == 200 + + resp = Couch.put("/#{db}/doc", body: %{"no" => "type"}) + assert resp.status_code == 201 + end + + @tag :with_db + test "Mango VDU rejects a doc if any existing ddoc fails to match", context do + db = context[:db_name] + resp = Couch.put("/#{db}/_design/mango-test", body: @mango_type_check) + assert resp.status_code == 201 + + ddoc = %{ + language: "query", + + validate_doc_update: %{ + "newDoc" => %{"year" => %{"$lt" => 2026}} + } + } + resp = Couch.put("/#{db}/_design/mango-test-2", body: ddoc) + assert resp.status_code == 201 + + resp = Couch.put("/#{db}/doc1", body: %{"type" => "movie", "year" => 1994}) + assert resp.status_code == 201 + + resp = Couch.put("/#{db}/doc2", body: %{"year" => 1994}) + assert resp.status_code == 403 + assert resp.body["error"] == "forbidden" + + resp = Couch.put("/#{db}/doc3", body: %{"type" => "movie", "year" => 2094}) + assert resp.status_code == 403 + assert resp.body["error"] == "forbidden" + end + + @tag :with_db + test "Mango VDU allows comparisons via $data", context do + db = context[:db_name] + + resp = Couch.put("/#{db}/_design/mango-test", body: %{ + language: "query", + + validate_doc_update: %{ + "$or" => [ + %{ "oldDoc" => :null }, + %{ "oldDoc.tags" => %{ "$size" => 0 } }, + %{ "newDoc.tags" => %{ "$all" => %{ "$data" => "oldDoc.tags" } } } + ] + } + }) + assert resp.status_code == 201 + + resp = Couch.put("/#{db}/doc", body: %{ + "tags" => ["a"] + }) + assert resp.status_code == 201 + rev = resp.body["rev"] + + resp = Couch.put("/#{db}/doc", query: %{rev: rev}, body: %{ + "tags" => ["a", "b"] + }) + assert resp.status_code == 201 + rev = resp.body["rev"] + + resp = Couch.put("/#{db}/doc", query: %{rev: rev}, body: %{ + "tags" => ["b", "a", "c"] + }) + assert resp.status_code == 201 + rev = resp.body["rev"] + + resp = Couch.put("/#{db}/doc", query: %{rev: rev}, body: %{ + "tags" => ["b", "c"] + }) + assert resp.status_code == 403 + end +end diff --git a/test/fixtures/allowed-xref.txt b/test/fixtures/allowed-xref.txt index c630fc109e1..8292d692b0f 100644 --- a/test/fixtures/allowed-xref.txt +++ b/test/fixtures/allowed-xref.txt @@ -1,2 +1,6 @@ +src/erlperf_cli.erl:{102,1}: Warning: erlperf_cli:main/1 calls undefined function args:format_error/3 (Xref) +src/erlperf_cli.erl:{102,1}: Warning: erlperf_cli:main/1 calls undefined function args:parse/3 (Xref) src/ioq.erl: Warning: ioq:get_disk_queues/0 is undefined function (Xref) src/weatherreport_check_ioq.erl:{95,1}: Warning: weatherreport_check_ioq:check_legacy_int/1 calls undefined function ioq:get_disk_queues/0 (Xref) +Warning: args:format_error/3 is undefined function (Xref) +Warning: args:parse/3 is undefined function (Xref)