Makes batch limit rollback to respect operation flags (#172)

* Makes batch limit rollback to respect operation flags

* Makes batch limit rollback to respect operation flags

In case of pre-processing repair it allows to rollback non-existent hold
operations.

* Moves try/catch deeper into turnover-limits iteration function

* Adds non-existent limit rollback testcase

* Fixes expected error term

* Fixes `repair_fail_routing_not_existent_operation` testcase

* Cleans up and adds assertion for limit-configs to exist in domain

* Updates testcase with exception assertion

* Allows non-existent limits rollback when repairing pre-processing

* Fixes missing args

Author: nanodirijabl

apps/hellgate/src/hg_invoice_payment.erl

@@ -2512,7 +2512,7 @@ get_limit_overflow_routes(Routes, VS, Iter, St) ->
         fun(Route, {RoutesNoOverflowIn, RejectedIn, LimitsIn}) ->
             PaymentRoute = hg_route:to_payment_route(Route),
             ProviderTerms = hg_routing:get_payment_terms(PaymentRoute, VS, Revision),
-            TurnoverLimits = get_turnover_limits(ProviderTerms),
+            TurnoverLimits = get_turnover_limits(ProviderTerms, strict),
             case hg_limiter:check_limits(TurnoverLimits, Invoice, Payment, PaymentRoute, Iter) of
                 {ok, Limits} ->
                     {[Route | RoutesNoOverflowIn], RejectedIn, LimitsIn#{PaymentRoute => Limits}};
@@ -2570,10 +2570,8 @@ rollback_shop_limits(Opts, St, Flags) ->
         Flags
     ).
 
-get_shop_turnover_limits(#domain_ShopConfig{turnover_limits = undefined}) ->
-    [];
-get_shop_turnover_limits(#domain_ShopConfig{turnover_limits = T}) ->
-    ordsets:to_list(T).
+get_shop_turnover_limits(ShopConfig) ->
+    hg_limiter:get_turnover_limits(ShopConfig, strict).
 
 %%
 
@@ -2588,7 +2586,7 @@ hold_limit_routes(Routes0, VS, Iter, St) ->
         fun(Route, {LimitHeldRoutes, RejectedRoutes} = Acc) ->
             PaymentRoute = hg_route:to_payment_route(Route),
             ProviderTerms = hg_routing:get_payment_terms(PaymentRoute, VS, Revision),
-            TurnoverLimits = get_turnover_limits(ProviderTerms),
+            TurnoverLimits = get_turnover_limits(ProviderTerms, strict),
             try
                 ok = hg_limiter:hold_payment_limits(TurnoverLimits, Invoice, Payment, PaymentRoute, Iter),
                 {[Route | LimitHeldRoutes], RejectedRoutes}
@@ -2622,7 +2620,7 @@ rollback_payment_limits(Routes, Iter, St, Flags) ->
     lists:foreach(
         fun(Route) ->
             ProviderTerms = hg_routing:get_payment_terms(Route, VS, Revision),
-            TurnoverLimits = get_turnover_limits(ProviderTerms),
+            TurnoverLimits = get_turnover_limits(ProviderTerms, strict),
             ok = hg_limiter:rollback_payment_limits(TurnoverLimits, Invoice, Payment, Route, Iter, Flags)
         end,
         Routes
@@ -2632,7 +2630,7 @@ rollback_broken_payment_limits(St) ->
     Opts = get_opts(St),
     Payment = get_payment(St),
     Invoice = get_invoice(Opts),
-    LimitValues = get_limit_values(St),
+    LimitValues = get_limit_values_(St, lenient),
     Iter = maps:size(LimitValues),
     maps:fold(
         fun
@@ -2661,9 +2659,8 @@ rollback_unused_payment_limits(St) ->
     UnUsedRoutes = Routes -- [Route],
     rollback_payment_limits(UnUsedRoutes, get_iter(St), St, [ignore_business_error, ignore_not_found]).
 
-get_turnover_limits(ProviderTerms) ->
-    TurnoverLimitSelector = ProviderTerms#domain_PaymentsProvisionTerms.turnover_limits,
-    hg_limiter:get_turnover_limits(TurnoverLimitSelector).
+get_turnover_limits(ProviderTerms, Mode) ->
+    hg_limiter:get_turnover_limits(ProviderTerms, Mode).
 
 commit_payment_limits(#st{capture_data = CaptureData} = St) ->
     Opts = get_opts(St),
@@ -2673,7 +2670,7 @@ commit_payment_limits(#st{capture_data = CaptureData} = St) ->
     Invoice = get_invoice(Opts),
     Route = get_route(St),
     ProviderTerms = get_provider_terms(St, Revision),
-    TurnoverLimits = get_turnover_limits(ProviderTerms),
+    TurnoverLimits = get_turnover_limits(ProviderTerms, strict),
     Iter = get_iter(St),
     hg_limiter:commit_payment_limits(TurnoverLimits, Invoice, Payment, Route, Iter, CapturedCash).
 
@@ -3450,8 +3447,11 @@ accrue_status_timing(Name, Opts, #st{timings = Timings}) ->
     EventTime = define_event_timestamp(Opts),
     hg_timings:mark(Name, EventTime, hg_timings:accrue(Name, started, EventTime, Timings)).
 
--spec get_limit_values(st()) -> route_limit_context().
-get_limit_values(St) ->
+-spec get_limit_values(st(), opts()) -> route_limit_context().
+get_limit_values(St, Opts) ->
+    get_limit_values_(St#st{opts = Opts}, strict).
+
+get_limit_values_(St, Mode) ->
     {PaymentInstitution, VS, Revision} = route_args(St),
     Ctx = build_routing_context(PaymentInstitution, VS, Revision, St),
     Payment = get_payment(St),
@@ -3468,20 +3468,15 @@ get_limit_values(St) ->
         fun(Route, Acc) ->
             PaymentRoute = hg_route:to_payment_route(Route),
             ProviderTerms = hg_routing:get_payment_terms(PaymentRoute, VS, Revision),
-            TurnoverLimits = get_turnover_limits(ProviderTerms),
-            TurnoverLimitValues = hg_limiter:get_limit_values(
-                TurnoverLimits, Invoice, Payment, PaymentRoute, Iter
-            ),
+            TurnoverLimits = get_turnover_limits(ProviderTerms, Mode),
+            TurnoverLimitValues =
+                hg_limiter:get_limit_values(TurnoverLimits, Invoice, Payment, PaymentRoute, Iter),
             Acc#{PaymentRoute => TurnoverLimitValues}
         end,
         #{},
         hg_routing_ctx:considered_candidates(Ctx)
     ).
 
--spec get_limit_values(st(), opts()) -> route_limit_context().
-get_limit_values(St, Opts) ->
-    get_limit_values(St#st{opts = Opts}).
-
 try_accrue_waiting_timing(Opts, #st{payment = Payment, timings = Timings}) ->
     case get_payment_flow(Payment) of
         ?invoice_payment_flow_instant() ->

apps/hellgate/src/hg_invoice_payment_refund.erl

@@ -422,8 +422,7 @@ get_resource_payment_tool(#domain_DisposablePaymentResource{payment_tool = Payme
     PaymentTool.
 
 get_turnover_limits(ProviderTerms) ->
-    TurnoverLimitSelector = ProviderTerms#domain_PaymentsProvisionTerms.turnover_limits,
-    hg_limiter:get_turnover_limits(TurnoverLimitSelector).
+    hg_limiter:get_turnover_limits(ProviderTerms, strict).
 
 prepare_refund_cashflow(Refund) ->
     hg_accounting:hold(construct_refund_plan_id(Refund), make_batch(Refund)).

apps/hellgate/src/hg_invoice_registered_payment.erl

@@ -215,8 +215,7 @@ get_turnover_limits(Payment, Route, St) ->
     RiskScore = hg_invoice_payment:get_risk_score(St),
     VS = collect_validation_varset(PartyConfigRef, ShopObj, Cost, PaymentTool, RiskScore),
     ProviderTerms = hg_routing:get_payment_terms(Route, VS, Revision),
-    TurnoverLimitSelector = ProviderTerms#domain_PaymentsProvisionTerms.turnover_limits,
-    hg_limiter:get_turnover_limits(TurnoverLimitSelector).
+    hg_limiter:get_turnover_limits(ProviderTerms, strict).
 
 construct_payment(
     PaymentID,

apps/hellgate/src/hg_limiter.erl

@@ -7,7 +7,7 @@
 -include_lib("limiter_proto/include/limproto_limiter_thrift.hrl").
 -include_lib("limiter_proto/include/limproto_context_payproc_thrift.hrl").
 
--type turnover_selector() :: dmsl_domain_thrift:'TurnoverLimitSelector'().
+-type turnover_terms_container() :: dmsl_domain_thrift:'PaymentsProvisionTerms'() | dmsl_domain_thrift:'ShopConfig'().
 -type turnover_limit() :: dmsl_domain_thrift:'TurnoverLimit'().
 -type invoice() :: dmsl_domain_thrift:'Invoice'().
 -type payment() :: dmsl_domain_thrift:'InvoicePayment'().
@@ -23,7 +23,7 @@
 
 -export_type([turnover_limit_value/0]).
 
--export([get_turnover_limits/1]).
+-export([get_turnover_limits/2]).
 -export([check_limits/5]).
 -export([check_shop_limits/5]).
 -export([hold_payment_limits/5]).
@@ -42,14 +42,56 @@
     terminal = TerminalRef
 }).
 
--spec get_turnover_limits(turnover_selector() | undefined) -> [turnover_limit()].
-get_turnover_limits(undefined) ->
+-define(party(PartyID), #domain_Party{
+    id = PartyID
+}).
+
+-define(shop(ShopID), #domain_Shop{
+    id = ShopID
+}).
+
+%% Very specific errors to crutch around
+-define(POSTING_PLAN_NOT_FOUND(ID), #base_InvalidRequest{errors = [<<"Posting plan not found: ", ID/binary>>]}).
+-define(OPERATION_NOT_FOUND, {invalid_request, [<<"OperationNotFound">>]}).
+
+-spec get_turnover_limits(turnover_terms_container(), strict | lenient) -> [turnover_limit()].
+
+get_turnover_limits(#domain_ShopConfig{turnover_limits = undefined}, _Mode) ->
+    [];
+get_turnover_limits(#domain_ShopConfig{turnover_limits = Limits}, Mode) ->
+    ordsets:to_list(filter_existing_turnover_limits(Limits, Mode));
+get_turnover_limits(#domain_PaymentsProvisionTerms{turnover_limits = undefined}, _Mode) ->
     [];
-get_turnover_limits({value, Limits}) ->
-    Limits;
-get_turnover_limits(Ambiguous) ->
+get_turnover_limits(#domain_PaymentsProvisionTerms{turnover_limits = {value, Limits}}, Mode) ->
+    filter_existing_turnover_limits(Limits, Mode);
+get_turnover_limits(#domain_PaymentsProvisionTerms{turnover_limits = Ambiguous}, _Mode) ->
     error({misconfiguration, {'Could not reduce selector to a value', Ambiguous}}).
 
+-define(LIMIT_NOT_FOUND(Revision, LimitID),
+    {object_not_found, {Revision, {limit_config, #domain_LimitConfigRef{id = LimitID}}}}
+).
+filter_existing_turnover_limits(Limits, Mode) ->
+    %% When mode is strict and limit-config does not exist it raises a
+    %% misconfiguration error.
+    %% Otherwise it filters out non existent one.
+    lists:filter(
+        fun
+            (#domain_TurnoverLimit{domain_revision = undefined}) ->
+                true;
+            (#domain_TurnoverLimit{id = ID, domain_revision = Ver}) ->
+                try
+                    _ = hg_domain:get(Ver, {limit_config, #domain_LimitConfigRef{id = ID}}),
+                    true
+                catch
+                    error:?LIMIT_NOT_FOUND(_Revision, _LimitID) when Mode =:= lenient ->
+                        false;
+                    error:?LIMIT_NOT_FOUND(Revision, LimitID) when Mode =:= strict ->
+                        error({misconfiguration, {'Limit config not found', {Revision, LimitID}}})
+                end
+        end,
+        Limits
+    ).
+
 -spec get_limit_values([turnover_limit()], invoice(), payment(), route(), pos_integer()) -> [turnover_limit_value()].
 get_limit_values(TurnoverLimits, Invoice, Payment, Route, Iter) ->
     Context = gen_limit_context(Invoice, Payment, Route),
@@ -277,13 +319,19 @@ rollback_payment_limits(TurnoverLimits, Invoice, Payment, Route, Iter, Flags) ->
     {LegacyTurnoverLimits, BatchTurnoverLimits} = split_turnover_limits_by_available_limiter_api(TurnoverLimits),
     ok = legacy_rollback_payment_limits(Context, LegacyTurnoverLimits, Invoice, Payment, Route, Iter, Flags),
     OperationIdSegments = make_route_operation_segments(Invoice, Payment, Route, Iter),
-    ok = batch_rollback_limits(Context, BatchTurnoverLimits, OperationIdSegments).
+    ok = batch_rollback_limits(Context, BatchTurnoverLimits, OperationIdSegments, Flags).
 
-batch_rollback_limits(_Context, [], _OperationIdSegments) ->
+batch_rollback_limits(_Context, [], _OperationIdSegments, _Flags) ->
     ok;
-batch_rollback_limits(Context, TurnoverLimits, OperationIdSegments) ->
+batch_rollback_limits(Context, TurnoverLimits, OperationIdSegments, Flags) ->
     {LimitRequest, _} = prepare_limit_request(TurnoverLimits, OperationIdSegments),
-    hg_limiter_client:rollback_batch(LimitRequest, Context).
+    IgnoreError = lists:member(ignore_not_found, Flags) orelse lists:member(ignore_business_error, Flags),
+    try
+        ok = hg_limiter_client:rollback_batch(LimitRequest, Context)
+    catch
+        error:(?OPERATION_NOT_FOUND) when IgnoreError =:= true ->
+            ok
+    end.
 
 legacy_rollback_payment_limits(Context, TurnoverLimits, Invoice, Payment, Route, Iter, Flags) ->
     ChangeIDs = [
@@ -308,7 +356,7 @@ rollback_shop_limits(TurnoverLimits, PartyConfigRef, ShopConfigRef, Invoice, Pay
         Context, LegacyTurnoverLimits, PartyConfigRef, ShopConfigRef, Invoice, Payment, Flags
     ),
     OperationIdSegments = make_shop_operation_segments(PartyConfigRef, ShopConfigRef, Invoice, Payment),
-    ok = batch_rollback_limits(Context, BatchTurnoverLimits, OperationIdSegments).
+    ok = batch_rollback_limits(Context, BatchTurnoverLimits, OperationIdSegments, Flags).
 
 legacy_rollback_shop_limits(Context, TurnoverLimits, PartyConfigRef, ShopConfigRef, Invoice, Payment, Flags) ->
     ChangeIDs = [construct_shop_change_id(PartyConfigRef, ShopConfigRef, Invoice, Payment)],
@@ -321,7 +369,7 @@ rollback_refund_limits(TurnoverLimits, Invoice, Payment, Refund, Route) ->
     {LegacyTurnoverLimits, BatchTurnoverLimits} = split_turnover_limits_by_available_limiter_api(TurnoverLimits),
     ok = legacy_rollback_refund_limits(Context, LegacyTurnoverLimits, Invoice, Payment, Refund),
     OperationIdSegments = make_refund_operation_segments(Invoice, Payment, Refund),
-    ok = batch_rollback_limits(Context, BatchTurnoverLimits, OperationIdSegments).
+    ok = batch_rollback_limits(Context, BatchTurnoverLimits, OperationIdSegments, []).
 
 legacy_rollback_refund_limits(Context, TurnoverLimits, Invoice, Payment, Refund) ->
     ChangeIDs = [construct_refund_change_id(Invoice, Payment, Refund)],
@@ -348,9 +396,6 @@ process_changes(LimitChangesQueues, WithFun, Clock, Context, Flags) ->
         LimitChangesQueues
     ).
 
-%% Very specific error to crutch around
--define(POSTING_PLAN_NOT_FOUND(ID), #base_InvalidRequest{errors = [<<"Posting plan not found: ", ID/binary>>]}).
-
 process_changes_try_wrap([LimitChange], WithFun, Clock, Context, Flags) ->
     IgnoreNotFound = lists:member(ignore_not_found, Flags),
     #limiter_LimitChange{change_id = ChangeID} = LimitChange,

apps/hellgate/src/hg_limiter_client.erl

@@ -30,8 +30,8 @@
 
 -spec get(limit_id(), limit_version() | undefined, clock(), context()) -> limit() | no_return().
 get(LimitID, Version, Clock, Context) ->
-    Args = {LimitID, Version, Clock, Context},
     Opts = hg_woody_wrapper:get_service_options(limiter),
+    Args = {LimitID, Version, Clock, Context},
     case hg_woody_wrapper:call(limiter, 'GetVersioned', Args, Opts) of
         {ok, Limit} ->
             Limit;

apps/hellgate/test/hg_invoice_tests_SUITE.erl

@@ -181,6 +181,7 @@
 -export([repair_fulfill_session_on_captured_succeeded/1]).
 
 -export([repair_fail_routing_succeeded/1]).
+-export([repair_fail_routing_not_existent_operation/1]).
 -export([repair_fail_cash_flow_building_succeeded/1]).
 
 -export([consistent_account_balances/1]).
@@ -472,6 +473,7 @@ groups() ->
         ]},
         {repair_preproc_w_limits, [], [
             repair_fail_routing_succeeded,
+            repair_fail_routing_not_existent_operation,
             repair_fail_cash_flow_building_succeeded
         ]},
         {route_cascading, [parallel], [
@@ -515,7 +517,6 @@ init_per_suite(C) ->
     ]),
 
     BaseLimitsRevision = hg_limiter_helper:init_per_suite(C),
-    _ = logger:error("BaseLimitsRevision: ~p", [BaseLimitsRevision]),
 
     RootUrl = maps:get(hellgate_root_url, Ret),
 
@@ -724,6 +725,8 @@ init_per_testcase(Name = repair_fail_routing_succeeded, C) ->
         fun override_check_limits/5
     ),
     init_per_testcase_(Name, C);
+init_per_testcase(Name = repair_fail_routing_not_existent_operation, C) ->
+    init_per_testcase_(Name, override_terms_limit_reference(?prv(5), C));
 init_per_testcase(Name = repair_fail_cash_flow_building_succeeded, C) ->
     meck:expect(
         hg_cashflow_utils,
@@ -756,6 +759,27 @@ override_domain_fixture(Fixture, C) ->
 override_domain_fixture(Fixture, Name, C) ->
     init_per_testcase_(Name, override_domain_fixture(Fixture, C)).
 
+override_terms_limit_reference(ProviderRef, C) ->
+    override_domain_fixture(
+        fun(Revision, _C) ->
+            [
+                change_provider_payments_provision_terms(ProviderRef, Revision, fun(PaymentsProvisionTerms) ->
+                    PaymentsProvisionTerms#domain_PaymentsProvisionTerms{
+                        turnover_limits =
+                            {value, [
+                                #domain_TurnoverLimit{
+                                    id = <<"NOT_EXISTENT_LIMIT_ID">>,
+                                    upper_boundary = ?LIMIT_UPPER_BOUNDARY,
+                                    domain_revision = Revision
+                                }
+                            ]}
+                    }
+                end)
+            ]
+        end,
+        C
+    ).
+
 init_per_testcase_(Name, C) ->
     ApiClient = hg_ct_helper:create_client(cfg(root_url, C)),
     Client = hg_client_invoicing:start_link(ApiClient),
@@ -774,6 +798,8 @@ trace_testcase(Name, C) ->
 end_per_testcase(repair_fail_routing_succeeded, C) ->
     meck:unload(hg_limiter),
     end_per_testcase(default, C);
+end_per_testcase(repair_fail_routing_not_existent_operation, C) ->
+    end_per_testcase(default, C);
 end_per_testcase(repair_fail_cash_flow_building_succeeded, C) ->
     meck:unload(hg_cashflow_utils),
     end_per_testcase(default, C);
@@ -1349,11 +1375,6 @@ payment_limit_overflow(C) ->
     ) = create_payment(PartyConfigRef, ShopConfigRef, PaymentAmount, Client, PmtSys),
 
     Failure = create_payment_limit_overflow(PartyConfigRef, ShopConfigRef, 1000, Client, PmtSys),
-    _ = logger:error("configured_limit_version(?LIMIT_ID, C): ~p", [configured_limit_version(?LIMIT_ID, C)]),
-    Res = dmt_client:checkout_object(
-        configured_limit_version(?LIMIT_ID, C), {limit_config, #domain_LimitConfigRef{id = ?LIMIT_ID}}
-    ),
-    _ = logger:error("dmt_client:checkout_object({limit_config, #domain_LimitConfigRef{id = ?LIMIT_ID}}: ~p", [Res]),
     ok = hg_limiter_helper:assert_payment_limit_amount(
         ?LIMIT_ID, configured_limit_version(?LIMIT_ID, C), PaymentAmount, Payment, Invoice
     ),
@@ -5715,6 +5736,43 @@ repair_fail_routing_succeeded(C) ->
         InvoiceID, fail_pre_processing, Client
     ).
 
+-spec repair_fail_routing_not_existent_operation(config()) -> test_return().
+repair_fail_routing_not_existent_operation(C) ->
+    RootUrl = cfg(root_url, C),
+    Client = hg_client_invoicing:start_link(hg_ct_helper:create_client(RootUrl)),
+    PartyClient = cfg(party_client, C),
+    #{party_config_ref := PartyConfigRef} = cfg(limits, C),
+    ShopConfigRef = hg_ct_helper:create_shop(PartyConfigRef, ?cat(8), <<"RUB">>, ?trms(1), ?pinst(1), PartyClient),
+
+    %% Invoice
+    InvoiceParams =
+        make_invoice_params(PartyConfigRef, ShopConfigRef, <<"rubberduck">>, make_due_date(10), make_cash(10000)),
+    InvoiceID = create_invoice(InvoiceParams, Client),
+    ?invoice_created(?invoice_w_status(?invoice_unpaid())) = next_change(InvoiceID, Client),
+
+    %% Payment
+    PaymentParams = make_payment_params(?pmt_sys(<<"visa-ref">>)),
+    ?payment_state(?payment(PaymentID)) = hg_client_invoicing:start_payment(InvoiceID, PaymentParams, Client),
+    [
+        ?payment_ev(PaymentID, ?payment_started(?payment_w_status(?pending()))),
+        ?payment_ev(PaymentID, ?shop_limit_initiated()),
+        ?payment_ev(PaymentID, ?shop_limit_applied()),
+        ?payment_ev(PaymentID, ?risk_score_changed(_))
+    ] = next_changes(InvoiceID, 4, Client),
+    %% Routing broken: limit holds fail with misconfiguration error
+    timeout = next_change(InvoiceID, 2000, Client),
+
+    %% Repair with rollback limits
+    ok = repair_invoice_with_scenario(InvoiceID, fail_pre_processing, Client),
+
+    %% Check final status
+    ?payment_ev(PaymentID, ?payment_status_changed(?failed({failure, _Failure}))) = next_change(InvoiceID, Client),
+
+    %% Check duplicate repair
+    {exception, {base_InvalidRequest, [<<"No need to repair">>]}} = repair_invoice_with_scenario(
+        InvoiceID, fail_pre_processing, Client
+    ).
+
 %% fail cash_flow_building before accounting hold
 -spec repair_fail_cash_flow_building_succeeded(config()) -> test_return().
 repair_fail_cash_flow_building_succeeded(C) ->