Skip to content

Commit bcc806d

Browse files
ferdferd
authored
Merge pull request #50 from ferd/ncurses-fafo-delete-fix-rebased
Add a whole ncurses TUI and fix a delete logic bug
2 parents 3775cd6 + a154d10 commit bcc806d

17 files changed

Lines changed: 1736 additions & 6 deletions

‎apps/revault/src/revault_data_wrapper.erl‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@
1111
-module(revault_data_wrapper).
1212
-export([peer/1, peer/2, new/0, ask/0, ok/0, error/1, fork/2]).
1313
-export([manifest/0, manifest/1,
14-
send_file/4, send_multipart_file/6, send_deleted/2,
14+
send_file/4, send_multipart_file/6,
15+
send_deleted/2, send_conflict_deleted/3,
1516
send_conflict_file/5, send_conflict_multipart_file/7, fetch_file/1,
1617
sync_complete/0]).
1718

@@ -73,6 +74,9 @@ send_multipart_file(Path, Vsn, Hash, M, N, Bin) when M >= 1, M =< N ->
7374
send_deleted(Path, Vsn) ->
7475
{deleted_file, ?VSN, Path, {Vsn, deleted}}.
7576

77+
send_conflict_deleted(WorkPath, ConflictsLeft, Meta) ->
78+
{conflict_file, ?VSN, WorkPath, deleted, ConflictsLeft, Meta}.
79+
7680
send_conflict_file(WorkPath, Path, ConflictsLeft, Meta, Bin) ->
7781
{conflict_file, ?VSN, WorkPath, Path, ConflictsLeft, Meta, Bin}.
7882

‎apps/revault/src/revault_data_wrapper.hrl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
% VSN 1: initial protocol
22
% VSN 2: adds multipart file transfers
33
% TODO: add test about protocol compatibility
4-
-define(VSN, 2).
4+
-define(VSN, 3).

‎apps/revault/src/revault_dirmon_tracker.erl‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,14 @@ handle_call({conflict, Work, {NewStamp, deleted}}, _From,
151151
%% but note the deletion stamp as part of the conflict.
152152
CStamp = conflict_stamp(Id, Stamp, NewStamp),
153153
{CStamp, {conflict, ConflictHashes, WorkingHash}};
154+
#{Work := {Stamp, deleted}} ->
155+
%% This is a special case similar to having both files diverging
156+
%% in stamps but having the same "hash" or value by virtue of being
157+
%% deleted. Create an empty conflict file, assume further files might
158+
%% come in as part of the sync or that this will properly carry
159+
%% the state moving forward.
160+
CStamp = conflict_stamp(Id, Stamp, NewStamp),
161+
{CStamp, {conflict, [], deleted}};
154162
#{Work := {Stamp, WorkingHash}} ->
155163
%% No conflict, create it
156164
ConflictingWork = revault_conflict_file:conflicting(Work, WorkingHash),
@@ -374,8 +382,10 @@ conflict_marker(Dir, WorkingFile) ->
374382
write_conflict_marker(Dir, WorkingFile, {_, {conflict, Hashes, _}}) ->
375383
%% We don't care about the rename trick here, it's informational
376384
%% but all the critical data is tracked in the snapshot
385+
F = conflict_marker(Dir, WorkingFile),
386+
revault_file:ensure_dir(F),
377387
revault_file:write_file(
378-
conflict_marker(Dir, WorkingFile),
388+
F,
379389
lists:join($\n, [revault_conflict_file:hex(Hash) || Hash <- Hashes])
380390
).
381391

‎apps/revault/src/revault_disterl.erl‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,8 @@ unpack({file, ?VSN, Path, Meta, PartNum, PartTotal, Bin}) -> {file, Path, Meta,
8383
unpack({fetch, ?VSN, Path}) -> {fetch, Path};
8484
unpack({sync_complete, ?VSN}) -> sync_complete;
8585
unpack({deleted_file, ?VSN, Path, Meta}) -> {deleted_file, Path, Meta};
86+
unpack({conflict_file, ?VSN, WorkPath, deleted, Count, Meta}) ->
87+
{conflict_file, WorkPath, deleted, Count, Meta};
8688
unpack({conflict_file, ?VSN, WorkPath, Path, Count, Meta, Bin}) ->
8789
{conflict_file, WorkPath, Path, Count, Meta, Bin};
8890
unpack({conflict_multipart_file, ?VSN, WorkPath, Path, Count, Meta, PartNum, PartTotal, Bin}) ->

‎apps/revault/src/revault_fsm.erl‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -637,6 +637,25 @@ client_sync_files(info, {revault, _Marker, {deleted_file, F, Meta}}, Data) ->
637637
NewQ = send_next_scheduled(Q),
638638
NewAcc = Acc -- [F],
639639
{keep_state, Data#data{scan=true, sub=S#client_sync{queue=NewQ, acc=NewAcc}}};
640+
client_sync_files(info, {revault, _Marker, {conflict_file, WorkF, deleted, CountLeft, Meta}}, Data) ->
641+
#data{name=Name, sub=S=#client_sync{queue=Q, acc=Acc}} = Data,
642+
?with_span(
643+
<<"conflict">>,
644+
#{attributes => [{<<"path">>, WorkF}, {<<"meta">>, ?str(Meta)},
645+
{<<"count">>, CountLeft} | ?attrs(Data)]},
646+
fun(_SpanCtx) ->
647+
%% TODO: handle the file being corrupted vs its own hash
648+
revault_dirmon_tracker:conflict(Name, WorkF, Meta)
649+
end
650+
),
651+
case CountLeft =:= 0 andalso Acc -- [WorkF] of
652+
false ->
653+
%% more of the same conflict file to come
654+
{keep_state, Data#data{scan=true}};
655+
NewAcc ->
656+
NewQ = send_next_scheduled(Q),
657+
{keep_state, Data#data{scan=true, sub=S#client_sync{queue=NewQ, acc=NewAcc}}}
658+
end;
640659
client_sync_files(info, {revault, _Marker, {conflict_file, WorkF, F, CountLeft, Meta, Bin}}, Data) ->
641660
#data{name=Name, sub=S=#client_sync{queue=Q, acc=Acc}} = Data,
642661
?with_span(
@@ -854,6 +873,16 @@ server_sync_files(info, {revault, _Marker, {deleted_file, F, Meta}},
854873
| ?attrs(Data)]},
855874
fun(_SpanCtx) -> handle_delete_sync(Name, Id, F, Meta) end),
856875
{keep_state, Data#data{scan=true}};
876+
server_sync_files(info, {revault, _M, {conflict_file, WorkF, deleted, CountLeft, Meta}}, Data) ->
877+
?with_span(
878+
<<"conflict">>,
879+
#{attributes => [{<<"path">>, WorkF}, {<<"meta">>, ?str(Meta)},
880+
{<<"count">>, CountLeft} | ?attrs(Data)]},
881+
fun(_SpanCtx) ->
882+
revault_dirmon_tracker:conflict(Data#data.name, WorkF, Meta)
883+
end
884+
),
885+
{keep_state, Data#data{scan=true}};
857886
server_sync_files(info, {revault, _M, {conflict_file, WorkF, F, CountLeft, Meta, Bin}}, Data) ->
858887
%% TODO: handle the file being corrupted vs its own hash
859888
?with_span(
@@ -1202,6 +1231,11 @@ file_transfer_schedule(Name, Path, File) ->
12021231
case revault_dirmon_tracker:file(Name, File) of
12031232
{Vsn, deleted} ->
12041233
[{deleted, File, Vsn}];
1234+
{Vsn, {conflict, [], deleted}} ->
1235+
%% Special deletion case where clashing deleted files
1236+
%% exist; there's no hash to send here, and no FHash;
1237+
%% explicitly call it as deleted.
1238+
[{conflict_file, File, deleted, 0, {Vsn, deleted}}];
12051239
{Vsn, {conflict, Hashes, _}} ->
12061240
{List, _} = lists:foldl(
12071241
fun(Hash, {Acc, Ct}) ->
@@ -1231,6 +1265,11 @@ wrap(_Path, {deleted, File, Vsn}) ->
12311265
?set_attribute(<<"path">>, File),
12321266
?set_attribute(<<"transfer_type">>, <<"deleted">>),
12331267
revault_data_wrapper:send_deleted(File, Vsn);
1268+
wrap(_Path, {conflict_file, File, deleted, Ct, Meta}) ->
1269+
?set_attribute(<<"path">>, deleted),
1270+
?set_attribute(<<"transfer_type">>, <<"conflict_file">>),
1271+
?set_attribute(<<"conflict.ct">>, Ct),
1272+
revault_data_wrapper:send_conflict_deleted(File, Ct, Meta);
12341273
wrap(Path, {conflict_file, File, FHash, Ct, Meta}) ->
12351274
?set_attribute(<<"path">>, FHash),
12361275
?set_attribute(<<"transfer_type">>, <<"conflict_file">>),
@@ -1303,4 +1342,3 @@ pid_attrs() ->
13031342
proplists:get_value(minor_gcs,
13041343
proplists:get_value(garbage_collection, PidInfo))}
13051344
].
1306-

‎apps/revault/src/revault_tcp.erl‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,8 @@ unpack({file, ?VSN, Path, Meta, PartNum, PartTotal, Bin}) -> {file, Path, Meta,
134134
unpack({fetch, ?VSN, Path}) -> {fetch, Path};
135135
unpack({sync_complete, ?VSN}) -> sync_complete;
136136
unpack({deleted_file, ?VSN, Path, Meta}) -> {deleted_file, Path, Meta};
137+
unpack({conflict_file, ?VSN, WorkPath, deleted, Count, Meta}) ->
138+
{conflict_file, WorkPath, deleted, Count, Meta};
137139
unpack({conflict_file, ?VSN, WorkPath, Path, Count, Meta, Bin}) ->
138140
{conflict_file, WorkPath, Path, Count, Meta, Bin};
139141
unpack({conflict_multipart_file, ?VSN, WorkPath, Path, Count, Meta, PartNum, PartTotal, Bin}) ->

‎apps/revault/src/revault_tls.erl‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,8 @@ unpack({file, ?VSN, Path, Meta, PartNum, PartTotal, Bin}) -> {file, Path, Meta,
138138
unpack({fetch, ?VSN, Path}) -> {fetch, Path};
139139
unpack({sync_complete, ?VSN}) -> sync_complete;
140140
unpack({deleted_file, ?VSN, Path, Meta}) -> {deleted_file, Path, Meta};
141+
unpack({conflict_file, ?VSN, WorkPath, deleted, Count, Meta}) ->
142+
{conflict_file, WorkPath, deleted, Count, Meta};
141143
unpack({conflict_file, ?VSN, WorkPath, Path, Count, Meta, Bin}) ->
142144
{conflict_file, WorkPath, Path, Count, Meta, Bin};
143145
unpack({conflict_multipart_file, ?VSN, WorkPath, Path, Count, Meta, PartNum, PartTotal, Bin}) ->

‎apps/revault/test/revault_fsm_SUITE.erl‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ groups() ->
2222
fork_server_save, seed_fork, basic_sync,
2323
delete_sync, too_many_clients,
2424
overwrite_sync_clash, conflict_sync,
25+
delete_sync_conflict,
2526
prevent_server_clash,
2627
multipart, double_conflict]}].
2728

@@ -714,6 +715,70 @@ conflict_sync(Config) ->
714715
?assertEqual({ok, <<"sh2">>}, file:read_file(filename:join([ClientPath2, "shared.1C56416E"]))),
715716
ok.
716717

718+
delete_sync_conflict() ->
719+
[{doc, "A deletion conflict can be sync'd to a third party"},
720+
{timetrap, timer:seconds(5)}].
721+
delete_sync_conflict(Config) ->
722+
Client = ?config(name, Config),
723+
Server=?config(server, Config),
724+
Remote = (?config(peer, Config))(Server),
725+
ClientPath = ?config(path, Config),
726+
ServerPath = ?config(server_path, Config),
727+
{ok, _ServId1} = revault_fsm:id(Server),
728+
{ok, _} = revault_fsm_sup:start_fsm(
729+
?config(db_dir, Config),
730+
Client,
731+
ClientPath,
732+
?config(ignore, Config),
733+
?config(interval, Config),
734+
(?config(callback, Config))(Client)
735+
),
736+
ok = revault_fsm:client(Client),
737+
{ok, _ClientId} = revault_fsm:id(Client, Remote),
738+
%% Set up a second client; because of how config works in the test, it needs
739+
Client2 = Client ++ "_2",
740+
Priv = ?config(priv_dir, Config),
741+
DbDir2 = filename:join([Priv, "db_2"]),
742+
ClientPath2 = filename:join([Priv, "data", "client_2"]),
743+
filelib:ensure_dir(filename:join([DbDir2, "fakefile"])),
744+
filelib:ensure_dir(filename:join([ClientPath2, "fakefile"])),
745+
{ok, _} = revault_fsm_sup:start_fsm(DbDir2, Client2, ClientPath2,
746+
?config(ignore, Config), ?config(interval, Config),
747+
(?config(callback, Config))(Client2)),
748+
ok = revault_fsm:client(Client2),
749+
?assertMatch({ok, _}, revault_fsm:id(Client2, Remote)),
750+
%% now in initialized mode
751+
%% Write files
752+
ok = file:write_file(filename:join([ServerPath, "shared"]), "sh1"),
753+
ok = file:write_file(filename:join([ClientPath, "shared"]), "sh2"),
754+
%% Track em
755+
ok = revault_dirmon_event:force_scan(Client, 5000),
756+
ok = revault_dirmon_event:force_scan(Server, 5000),
757+
%% Delete em
758+
ok = file:delete(filename:join([ServerPath, "shared"])),
759+
ok = file:delete(filename:join([ClientPath, "shared"])),
760+
%% Track the deletion
761+
ok = revault_dirmon_event:force_scan(Client, 5000),
762+
ok = revault_dirmon_event:force_scan(Server, 5000),
763+
%% Sync em
764+
ct:pal("SYNC", []),
765+
ok = revault_fsm:sync(Client, Remote),
766+
%% See the result
767+
%% conflicting files are marked, with empty conflict files since nothing exists aside
768+
%% from clashing deletions.
769+
?assertEqual({error, enoent}, file:read_file(filename:join([ServerPath, "shared"]))),
770+
?assertEqual({error, enoent}, file:read_file(filename:join([ClientPath, "shared"]))),
771+
?assertEqual({ok, <<"">>}, file:read_file(filename:join([ServerPath, "shared.conflict"])) ),
772+
?assertEqual({ok, <<"">>}, file:read_file(filename:join([ClientPath, "shared.conflict"])) ),
773+
774+
%% Now when client 2 syncs, it gets the files and conflict files as well
775+
ct:pal("SECOND SYNC", []),
776+
ok = revault_fsm:sync(Client2, Remote),
777+
%% conflicting files are marked, but working files aren't sync'd since they didn't exist here
778+
?assertEqual({error, enoent}, file:read_file(filename:join([ClientPath2, "shared"]))),
779+
?assertEqual({ok, <<"">>}, file:read_file(filename:join([ClientPath2, "shared.conflict"])) ),
780+
ok.
781+
717782
prevent_server_clash() ->
718783
[{doc, "A client from a different server cannot connect to the wrong one "
719784
"as it is protected by a UUID."},

‎cli/revault_cli/rebar.config‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1-
{deps, [argparse]}.
1+
{deps, [argparse,
2+
{cecho, {git, "https://github.com/ferd/cecho.git", {branch, "master"}}}]}.

‎cli/revault_cli/src/revault_cli.app.src‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,12 @@
22
[{description, "An escript to interact with ReVault nodes"},
33
{vsn, "0.1.0"},
44
{registered, []},
5+
{mod, {revault_cli_app, []}},
56
{applications,
67
[kernel,
78
stdlib,
8-
argparse
9+
argparse,
10+
cecho
911
]},
1012
{env,[]},
1113
{modules, []},

0 commit comments

Comments
 (0)