Skip to content
This repository was archived by the owner on May 25, 2021. It is now read-only.

Commit f916079

Browse files
jiangphcnjiangphcn
committed
Update couch_mrview for clustered purge improvement
 - implementation of updating local purge document - provide verify_index_exists/1 to check whether index exists or not - clean up local purge document in cleanup cycle - add new test case and refine existing test cases - create local purge document if it doesn’t exist when couch_mrview_index:open/2 is called - directly use db to clean up local purge document - use couch_db:purge_docs/2 with UUID - add error handling for verify_index_exists/1 - add check for Sig in couch_mrview_cleanup:run/1 - construct get_signature_from_filename/1 - use utc_string/0 provided by couch_util.erl - use ddoc_cache:open/2 to open design document - extend test from couch_db to fabric Bugzid: 68276
1 parent 5b2de8d commit f916079

5 files changed

Lines changed: 257 additions & 54 deletions

‎src/couch_mrview_cleanup.erl‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,23 @@ run(Db) ->
4141

4242
lists:foreach(fun(FN) ->
4343
couch_log:debug("Deleting stale view file: ~s", [FN]),
44-
couch_file:delete(RootDir, FN, [sync])
44+
couch_file:delete(RootDir, FN, [sync]),
45+
Sig = couch_mrview_util:get_signature_from_filename(FN),
46+
if length(Sig) < 16 -> ok; true ->
47+
case re:run(Sig,"^[a-fA-F0-9]+$",[{capture, none}]) of
48+
match ->
49+
DocId = couch_mrview_util:get_local_purge_doc_id(Sig),
50+
case couch_db:open_doc(Db, DocId, []) of
51+
{ok, LocalPurgeDoc} ->
52+
couch_db:update_doc(Db,
53+
LocalPurgeDoc#doc{deleted=true}, [?ADMIN_CTX]);
54+
{not_found, _} ->
55+
ok
56+
end;
57+
_ ->
58+
ok
59+
end
60+
end
4561
end, ToDelete),
4662

4763
ok.

‎src/couch_mrview_index.erl‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
-export([start_update/3, purge/4, process_doc/3, finish_update/1, commit/1]).
1919
-export([compact/3, swap_compacted/2]).
2020
-export([index_file_exists/1]).
21+
-export([update_local_purge_doc/2, verify_index_exists/1]).
2122

2223
-include_lib("couch/include/couch_db.hrl").
2324
-include_lib("couch_mrview/include/couch_mrview.hrl").
@@ -134,14 +135,17 @@ open(Db, State) ->
134135
{ok, {OldSig, Header}} ->
135136
% Matching view signatures.
136137
NewSt = couch_mrview_util:init_state(Db, Fd, State, Header),
138+
maybe_create_local_purge_doc(Db, NewSt),
137139
{ok, NewSt};
138140
% end of upgrade code for <= 1.2.x
139141
{ok, {Sig, Header}} ->
140142
% Matching view signatures.
141143
NewSt = couch_mrview_util:init_state(Db, Fd, State, Header),
144+
maybe_create_local_purge_doc(Db, NewSt),
142145
{ok, NewSt};
143146
_ ->
144147
NewSt = couch_mrview_util:reset_index(Db, Fd, State),
148+
maybe_create_local_purge_doc(Db, NewSt),
145149
{ok, NewSt}
146150
end;
147151
{error, Reason} = Error ->
@@ -204,3 +208,60 @@ index_file_exists(State) ->
204208
} = State,
205209
IndexFName = couch_mrview_util:index_file(DbName, Sig),
206210
filelib:is_file(IndexFName).
211+
212+
213+
update_local_purge_doc(Db, State) ->
214+
Sig = couch_index_util:hexsig(get(signature, State)),
215+
Doc = couch_doc:from_json_obj({[
216+
{<<"_id">>, couch_mrview_util:get_local_purge_doc_id(Sig)},
217+
{<<"purge_seq">>, get(purge_seq, State)},
218+
{<<"timestamp_utc">>, list_to_binary(couch_util:utc_string())},
219+
{<<"verify_module">>, <<"couch_mrview_index">>},
220+
{<<"verify_function">>, <<"verify_index_exists">>},
221+
{<<"verify_options">>, {[
222+
{<<"dbname">>, get(db_name, State)},
223+
{<<"ddoc_id">>, get(idx_name, State)},
224+
{<<"signature">>, Sig}
225+
]}},
226+
{<<"type">>, <<"mrview">>}
227+
]}),
228+
couch_db:update_doc(Db, Doc, []).
229+
230+
231+
verify_index_exists(Options) ->
232+
ShardDbName = couch_mrview_util:get_value_from_options(<<"dbname">>, Options),
233+
DDocId = couch_mrview_util:get_value_from_options(<<"ddoc_id">>, Options),
234+
SigInLocal = couch_mrview_util:get_value_from_options(<<"signature">>, Options),
235+
case couch_db:open_int(ShardDbName, []) of
236+
{ok, Db} ->
237+
try
238+
DbName = mem3:dbname(Db#db.name),
239+
case ddoc_cache:open(DbName, DDocId) of
240+
{ok, DDoc} ->
241+
{ok, IdxState} = couch_mrview_util:ddoc_to_mrst(ShardDbName, DDoc),
242+
couch_index_util:hexsig(IdxState#mrst.sig) == SigInLocal;
243+
_Else ->
244+
false
245+
end
246+
catch E:T ->
247+
Stack = erlang:get_stacktrace(),
248+
couch_log:error("Error occurs when verifying existence of ~s/~s :: ~p ~p",
249+
[ShardDbName, DDocId, {E, T}, Stack]),
250+
false
251+
after
252+
catch couch_db:close(Db)
253+
end;
254+
_ ->
255+
false
256+
end.
257+
258+
259+
maybe_create_local_purge_doc(Db, State) ->
260+
Sig = couch_index_util:hexsig(get(signature, State)),
261+
LocalPurgeDocId = couch_mrview_util:get_local_purge_doc_id(Sig),
262+
case couch_db:open_doc(Db, LocalPurgeDocId, []) of
263+
{not_found, _Reason} ->
264+
update_local_purge_doc(Db, State);
265+
{ok, _LocalPurgeDoc} ->
266+
ok
267+
end.

‎src/couch_mrview_util.erl‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212

1313
-module(couch_mrview_util).
1414

15+
-export([get_local_purge_doc_id/1, get_value_from_options/2]).
16+
-export([get_signature_from_filename/1]).
1517
-export([get_view/4]).
1618
-export([ddoc_to_mrst/2, init_state/4, reset_index/3]).
1719
-export([make_header/1]).
@@ -39,6 +41,26 @@
3941
-include_lib("couch_mrview/include/couch_mrview.hrl").
4042

4143

44+
get_local_purge_doc_id(Sig) ->
45+
list_to_binary(?LOCAL_DOC_PREFIX ++ "purge-mrview-" ++ Sig).
46+
47+
48+
get_value_from_options(Key, Options) ->
49+
case couch_util:get_value(Key, Options) of
50+
undefined ->
51+
Reason = binary_to_list(Key) ++ " must exist in Options.",
52+
throw({bad_request, Reason});
53+
Value -> Value
54+
end.
55+
56+
57+
get_signature_from_filename(FileName) ->
58+
FilePathList = filename:split(FileName),
59+
[PureFN] = lists:nthtail(length(FilePathList) - 1, FilePathList),
60+
PureFNExt = filename:extension(PureFN),
61+
filename:basename(PureFN, PureFNExt).
62+
63+
4264
get_view(Db, DDoc, ViewName, Args0) ->
4365
ArgCheck = fun(InitState) ->
4466
Args1 = set_view_type(Args0, ViewName, InitState#mrst.views),

‎test/couch_mrview_purge_docs_fabric_tests.erl‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
2+
% use this file except in compliance with the License. You may obtain a copy of
3+
% the License at
4+
%
5+
% http://www.apache.org/licenses/LICENSE-2.0
6+
%
7+
% Unless required by applicable law or agreed to in writing, software
8+
% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9+
% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10+
% License for the specific language governing permissions and limitations under
11+
% the License.
12+
13+
-module(couch_mrview_purge_docs_fabric_tests).
14+
15+
-include_lib("couch/include/couch_eunit.hrl").
16+
-include_lib("couch/include/couch_db.hrl").
17+
-include_lib("couch_mrview/include/couch_mrview.hrl").
18+
19+
-define(TIMEOUT, 1000).
20+
21+
22+
setup() ->
23+
DbName = ?tempdb(),
24+
ok = fabric:create_db(DbName, [?ADMIN_CTX]),
25+
DbName.
26+
27+
28+
teardown(DbName) ->
29+
ok = fabric:delete_db(DbName, [?ADMIN_CTX]).
30+
31+
32+
view_purge_fabric_test_() ->
33+
{
34+
"Map views",
35+
{
36+
setup,
37+
fun() -> test_util:start_couch([fabric, mem3]) end,
38+
fun test_util:stop_couch/1,
39+
{
40+
foreach,
41+
fun setup/0, fun teardown/1,
42+
[
43+
fun test_purge_verify_index/1
44+
]
45+
}
46+
}
47+
}.
48+
49+
50+
test_purge_verify_index(DbName) ->
51+
?_test(begin
52+
Docs1 = couch_mrview_test_util:make_docs(5),
53+
{ok, _} = fabric:update_docs(DbName, Docs1, [?ADMIN_CTX]),
54+
{ok, _} = fabric:update_doc(DbName, couch_mrview_test_util:ddoc(map), [?ADMIN_CTX]),
55+
56+
purge_docs(DbName, [<<"1">>]),
57+
58+
Result2 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}),
59+
Expect2 = {ok, [
60+
{meta, [{total, 4}, {offset, 0}]},
61+
{row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
62+
{row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
63+
{row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
64+
{row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
65+
]},
66+
?assertEqual(Expect2, Result2),
67+
68+
{ok, DDoc} = fabric:open_doc(DbName, <<"_design/bar">>, []),
69+
{ok, IdxState} = couch_mrview_util:ddoc_to_mrst(DbName, DDoc),
70+
Sig = IdxState#mrst.sig,
71+
HexSig = list_to_binary(couch_index_util:hexsig(Sig)),
72+
DocId = couch_mrview_util:get_local_purge_doc_id(HexSig),
73+
{ok, LocPurgeDoc} = fabric:open_doc(DbName, DocId, []),
74+
{Props} = couch_doc:to_json_obj(LocPurgeDoc,[]),
75+
{Options} = couch_util:get_value(<<"verify_options">>, Props),
76+
?assertEqual(true, couch_mrview_index:verify_index_exists(Options)),
77+
78+
ok
79+
end).
80+
81+
get_rev(#full_doc_info{} = FDI) ->
82+
#doc_info{
83+
revs = [#rev_info{} = PrevRev | _]
84+
} = couch_doc:to_doc_info(FDI),
85+
PrevRev#rev_info.rev.
86+
87+
88+
purge_docs(DbName, DocIds) ->
89+
lists:foreach(fun(DocId) ->
90+
FDI = fabric:get_full_doc_info(DbName, DocId, []),
91+
Rev = get_rev(FDI),
92+
{ok, {_, [{ok, _}]}} = fabric:purge_docs(DbName, [{DocId, [Rev]}], [])
93+
end, DocIds).

‎test/couch_mrview_purge_docs_tests.erl‎

Lines changed: 64 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
-include_lib("couch/include/couch_eunit.hrl").
1616
-include_lib("couch/include/couch_db.hrl").
17+
-include_lib("couch_mrview/include/couch_mrview.hrl").
1718

1819
-define(TIMEOUT, 1000).
1920

@@ -46,62 +47,72 @@ view_purge_test_() ->
4647

4748

4849
test_purge_single(Db) ->
49-
Result = run_query(Db, []),
50-
Expect = {ok, [
51-
{meta, [{total, 5}, {offset, 0}]},
52-
{row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
53-
{row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
54-
{row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
55-
{row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
56-
{row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
57-
]},
58-
?_assertEqual(Expect, Result),
59-
60-
FDI = couch_db:get_full_doc_info(Db, <<"1">>),
61-
Rev = get_rev(FDI),
62-
{ok, {_, _}} = couch_db:purge_docs(Db, [{<<"1">>, [Rev]}]),
63-
{ok, Db2} = couch_db:reopen(Db),
64-
65-
Result2 = run_query(Db2, []),
66-
Expect2 = {ok, [
67-
{meta, [{total, 4}, {offset, 0}]},
68-
{row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
69-
{row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
70-
{row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
71-
{row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
72-
]},
73-
?_assertEqual(Expect2, Result2).
50+
?_test(begin
51+
Result = run_query(Db, []),
52+
Expect = {ok, [
53+
{meta, [{total, 5}, {offset, 0}]},
54+
{row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
55+
{row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
56+
{row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
57+
{row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
58+
{row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
59+
]},
60+
?assertEqual(Expect, Result),
61+
62+
FDI = couch_db:get_full_doc_info(Db, <<"1">>),
63+
Rev = get_rev(FDI),
64+
{ok, {_, _}} = couch_db:purge_docs(Db, [{<<"UUID1">>, <<"1">>, [Rev]}]),
65+
{ok, Db2} = couch_db:reopen(Db),
66+
67+
Result2 = run_query(Db2, []),
68+
Expect2 = {ok, [
69+
{meta, [{total, 4}, {offset, 0}]},
70+
{row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
71+
{row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
72+
{row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
73+
{row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
74+
]},
75+
?assertEqual(Expect2, Result2),
76+
77+
ok
78+
end).
7479

7580

7681
test_purge_multiple(Db) ->
77-
Result = run_query(Db, []),
78-
Expect = {ok, [
79-
{meta, [{total, 5}, {offset, 0}]},
80-
{row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
81-
{row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
82-
{row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
83-
{row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
84-
{row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
85-
]},
86-
?_assertEqual(Expect, Result),
87-
88-
% 1st purge request
89-
FDI1 = couch_db:get_full_doc_info(Db, <<"1">>), Rev1 = get_rev(FDI1),
90-
FDI2 = couch_db:get_full_doc_info(Db, <<"2">>), Rev2 = get_rev(FDI2),
91-
FDI5 = couch_db:get_full_doc_info(Db, <<"5">>), Rev5 = get_rev(FDI5),
92-
93-
IdsRevs = [{<<"1">>, [Rev1]}, {<<"2">>, [Rev2]}, {<<"5">>, [Rev5]}],
94-
{ok, {_, _}} = couch_db:purge_docs(Db, IdsRevs),
95-
{ok, Db2} = couch_db:reopen(Db),
96-
97-
Result2 = run_query(Db2, []),
98-
Expect2 = {ok, [
99-
{meta, [{total, 2}, {offset, 0}]},
100-
{row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
101-
{row, [{id, <<"4">>}, {key, 4}, {value, 4}]}
102-
]},
103-
?_assertEqual(Expect2, Result2).
104-
82+
?_test(begin
83+
Result = run_query(Db, []),
84+
Expect = {ok, [
85+
{meta, [{total, 5}, {offset, 0}]},
86+
{row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
87+
{row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
88+
{row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
89+
{row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
90+
{row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
91+
]},
92+
?assertEqual(Expect, Result),
93+
94+
FDI1 = couch_db:get_full_doc_info(Db, <<"1">>), Rev1 = get_rev(FDI1),
95+
FDI2 = couch_db:get_full_doc_info(Db, <<"2">>), Rev2 = get_rev(FDI2),
96+
FDI5 = couch_db:get_full_doc_info(Db, <<"5">>), Rev5 = get_rev(FDI5),
97+
98+
IdsRevs = [
99+
{<<"UUID1">>, <<"1">>, [Rev1]},
100+
{<<"UUID2">>, <<"2">>, [Rev2]},
101+
{<<"UUID5">>, <<"5">>, [Rev5]}
102+
],
103+
{ok, {_, _}} = couch_db:purge_docs(Db, IdsRevs),
104+
{ok, Db2} = couch_db:reopen(Db),
105+
106+
Result2 = run_query(Db2, []),
107+
Expect2 = {ok, [
108+
{meta, [{total, 2}, {offset, 0}]},
109+
{row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
110+
{row, [{id, <<"4">>}, {key, 4}, {value, 4}]}
111+
]},
112+
?assertEqual(Expect2, Result2),
113+
114+
ok
115+
end).
105116

106117
run_query(Db, Opts) ->
107118
couch_mrview:query_view(Db, <<"_design/bar">>, <<"baz">>, Opts).

0 commit comments

Comments
 (0)