ED-305: withdrawal repair tools

apps/ff_cth/src/ct_domain_config.erl

@@ -6,6 +6,8 @@
 
 -export([head/0]).
 
+-export([all/1]).
+-export([checkout_object/2]).
 -export([commit/2]).
 -export([insert/1]).
 -export([update/1]).
@@ -21,6 +23,7 @@
 
 -type revision() :: dmt_client:version().
 -type object() :: dmsl_domain_thrift:'DomainObject'().
+-type object_ref() :: dmt_client:object_ref().
 
 -spec head() -> revision().
 head() ->
@@ -31,6 +34,10 @@ all(Revision) ->
     #'Snapshot'{domain = Domain} = dmt_client:checkout(Revision),
     Domain.
 
+-spec checkout_object(revision(), object_ref()) -> object() | no_return().
+checkout_object(Revision, ObjectRef) ->
+    dmt_client:checkout_object(Revision, ObjectRef).
+
 -spec commit(revision(), dmt_client:commit()) -> revision() | no_return().
 commit(Revision, Commit) ->
     dmt_client:commit(Revision, Commit).

apps/ff_server/src/ff_server.erl

@@ -93,7 +93,7 @@ init([]) ->
             {withdrawal_session_management, ff_withdrawal_session_handler},
             {deposit_management, ff_deposit_handler},
             {withdrawal_session_repairer, ff_withdrawal_session_repair},
-            {withdrawal_repairer, ff_withdrawal_repair},
+            {withdrawal_repairer, ff_withdrawal_repair_handler},
             {deposit_repairer, ff_deposit_repair},
             {w2w_transfer_management, ff_w2w_transfer_handler},
             {w2w_transfer_repairer, ff_w2w_transfer_repair}

apps/ff_server/src/ff_withdrawal_codec.erl

@@ -203,6 +203,12 @@ unmarshal(repair_scenario, {add_events, #wthd_AddEventsRepair{events = Events, a
             events => unmarshal({list, change}, Events),
             action => maybe_unmarshal(complex_action, Action)
         })};
+unmarshal(repair_scenario, {routing, RoutingScenarioType}) ->
+    {routing, unmarshal(repair_scenario_routing, RoutingScenarioType)};
+unmarshal(repair_scenario_routing, {route_changed, #wthd_RoutingRepairRouteChanged{route = Route}}) ->
+    {route_changed, unmarshal(route, Route)};
+unmarshal(repair_scenario_routing, {route_not_found, #wthd_RoutingRepairRouteNotFound{}}) ->
+    route_not_found;
 unmarshal(change, {created, #wthd_CreatedChange{withdrawal = Withdrawal}}) ->
     {created, unmarshal(withdrawal, Withdrawal)};
 unmarshal(change, {status_changed, #wthd_StatusChange{status = Status}}) ->

apps/ff_server/src/ff_withdrawal_repair.erl → apps/ff_server/src/ff_withdrawal_repair_handler.erl

@@ -1,4 +1,4 @@
--module(ff_withdrawal_repair).
+-module(ff_withdrawal_repair_handler).
 
 -behaviour(ff_woody_wrapper).
 
@@ -21,5 +21,7 @@ handle_function('Repair', {ID, Scenario}, _Opts) ->
         {error, notfound} ->
             woody_error:raise(business, #fistful_WithdrawalNotFound{});
         {error, working} ->
-            woody_error:raise(business, #fistful_MachineAlreadyWorking{})
+            woody_error:raise(business, #fistful_MachineAlreadyWorking{});
+        {error, {failed, _Error}} ->
+            woody_error:raise(business, #fistful_RepairScenarioFailed{})
     end.

apps/ff_transfer/src/ff_withdrawal.erl

@@ -7,6 +7,8 @@
 -include_lib("damsel/include/dmsl_payment_processing_thrift.hrl").
 -include_lib("damsel/include/dmsl_withdrawals_provider_adapter_thrift.hrl").
 
+-include_lib("fistful_proto/include/ff_proto_withdrawal_thrift.hrl").
+
 -type id() :: binary().
 -type clock() :: ff_transaction:clock().
 
@@ -24,6 +26,7 @@
     attempts => attempts(),
     resource => destination_resource(),
     adjustments => adjustments_index(),
+    repair_scenario => repair_scenario(),
     status => status(),
     metadata => metadata(),
     external_id => id()
@@ -170,7 +173,7 @@
 -type invalid_withdrawal_status_error() ::
     {invalid_withdrawal_status, status()}.
 
--type action() :: sleep | continue | undefined.
+-type action() :: sleep | continue | undefined | {set_timer, integer()}.
 
 -export_type([withdrawal/0]).
 -export_type([withdrawal_state/0]).
@@ -204,6 +207,7 @@
 -export([id/1]).
 -export([body/1]).
 -export([status/1]).
+-export([activity/1]).
 -export([route/1]).
 -export([attempts/1]).
 -export([external_id/1]).
@@ -228,6 +232,7 @@
 -export([session_id/1]).
 -export([get_current_session/1]).
 -export([get_current_session_status/1]).
+-export([repair_check_activity_compatibility/2]).
 
 %% Event source
 
@@ -260,6 +265,7 @@
 -type adjustment() :: ff_adjustment:adjustment().
 -type adjustment_id() :: ff_adjustment:id().
 -type adjustments_index() :: ff_adjustment_utils:index().
+-type repair_scenario() :: ff_repair:scenario() | undefined.
 -type currency_id() :: ff_currency:id().
 -type party_revision() :: ff_party:revision().
 -type domain_revision() :: ff_domain_config:revision().
@@ -355,6 +361,10 @@ status(T) ->
 route(T) ->
     maps:get(route, T, undefined).
 
+-spec activity(withdrawal_state()) -> activity().
+activity(T) ->
+    deduce_activity(T).
+
 -spec attempts(withdrawal_state()) -> attempts().
 attempts(#{attempts := Attempts}) ->
     Attempts;
@@ -542,7 +552,14 @@ is_finished(#{status := pending}) ->
 -spec process_transfer(withdrawal_state()) -> process_result().
 process_transfer(Withdrawal) ->
     Activity = deduce_activity(Withdrawal),
-    do_process_transfer(Activity, Withdrawal).
+    case Activity of
+        {fail, Reason} ->
+            process_route_change(Withdrawal, Reason);
+        _Other ->
+            RepairScenario = maps:get(repair_scenario, Withdrawal, undefined),
+            ProcessTransferFun = deduce_process_transfer(Activity),
+            ProcessTransferFun(Withdrawal, RepairScenario)
+    end.
 
 %%
 
@@ -673,6 +690,24 @@ operation_domain_revision(Withdrawal) ->
 
 %% Processing helpers
 
+-type transfer_process() :: fun((withdrawal_state()) -> process_result()).
+
+-spec deduce_process_transfer(activity()) -> transfer_process().
+deduce_process_transfer(Activity) ->
+    case Activity of
+        routing -> fun process_routing/2;
+        p_transfer_start -> fun process_p_transfer_creation/2;
+        p_transfer_prepare -> fun process_p_transfer_prepare/2;
+        p_transfer_commit -> fun process_p_transfer_commit/2;
+        p_transfer_cancel -> fun process_p_transfer_cancel/2;
+        limit_check -> fun process_limit_check/2;
+        session_starting -> fun process_session_creation/2;
+        session_sleeping -> fun process_session_sleep/2;
+        finish -> fun process_transfer_finish/2;
+        adjustment -> fun process_adjustment/2;
+        stop -> fun process_stop/2
+    end.
+
 -spec deduce_activity(withdrawal_state()) -> activity().
 deduce_activity(Withdrawal) ->
     Params = #{
@@ -729,40 +764,22 @@ do_finished_activity(#{status := succeeded, p_transfer := committed}) ->
 do_finished_activity(#{status := {failed, _}, p_transfer := cancelled}) ->
     stop.
 
--spec do_process_transfer(activity(), withdrawal_state()) -> process_result().
-do_process_transfer(routing, Withdrawal) ->
-    process_routing(Withdrawal);
-do_process_transfer(p_transfer_start, Withdrawal) ->
-    process_p_transfer_creation(Withdrawal);
-do_process_transfer(p_transfer_prepare, Withdrawal) ->
-    Tr = ff_withdrawal_route_attempt_utils:get_current_p_transfer(attempts(Withdrawal)),
-    {ok, Events} = ff_postings_transfer:prepare(Tr),
-    {continue, [{p_transfer, Ev} || Ev <- Events]};
-do_process_transfer(p_transfer_commit, Withdrawal) ->
-    Tr = ff_withdrawal_route_attempt_utils:get_current_p_transfer(attempts(Withdrawal)),
-    {ok, Events} = ff_postings_transfer:commit(Tr),
-    {continue, [{p_transfer, Ev} || Ev <- Events]};
-do_process_transfer(p_transfer_cancel, Withdrawal) ->
-    Tr = ff_withdrawal_route_attempt_utils:get_current_p_transfer(attempts(Withdrawal)),
-    {ok, Events} = ff_postings_transfer:cancel(Tr),
-    {continue, [{p_transfer, Ev} || Ev <- Events]};
-do_process_transfer(limit_check, Withdrawal) ->
-    process_limit_check(Withdrawal);
-do_process_transfer(session_starting, Withdrawal) ->
-    process_session_creation(Withdrawal);
-do_process_transfer(session_sleeping, Withdrawal) ->
-    process_session_sleep(Withdrawal);
-do_process_transfer({fail, Reason}, Withdrawal) ->
-    process_route_change(Withdrawal, Reason);
-do_process_transfer(finish, Withdrawal) ->
-    process_transfer_finish(Withdrawal);
-do_process_transfer(adjustment, Withdrawal) ->
-    process_adjustment(Withdrawal);
-do_process_transfer(stop, _Withdrawal) ->
+-spec repair_check_activity_compatibility(repair_scenario(), activity()) -> ok.
+repair_check_activity_compatibility({routing, _}, Activity) when Activity =:= routing ->
+    ok;
+repair_check_activity_compatibility(Scenario, Activity) ->
+    {error, {not_compatible, {activity, Activity}, {scenario, Scenario}}}.
+
+-spec process_stop(withdrawal_state(), repair_scenario()) -> process_result().
+process_stop(_Withdrawal, undefined) ->
     {undefined, []}.
 
--spec process_routing(withdrawal_state()) -> process_result().
-process_routing(Withdrawal) ->
+-spec process_routing(withdrawal_state(), repair_scenario()) -> process_result().
+process_routing(_Withdrawal, {routing, {route_changed, Route}}) ->
+    {continue, [{route_changed, Route}]};
+process_routing(Withdrawal, {routing, route_not_found}) ->
+    process_transfer_fail(route_not_found, Withdrawal);
+process_routing(Withdrawal, undefined) ->
     case do_process_routing(Withdrawal) of
         {ok, [Route | _]} ->
             {continue, [
@@ -825,8 +842,8 @@ validate_quote_terminal(#{terminal_id := TerminalID}, #{terminal_id := TerminalI
 validate_quote_terminal(#{terminal_id := TerminalID}, _) ->
     {error, {inconsistent_quote_route, {terminal_id, TerminalID}}}.
 
--spec process_limit_check(withdrawal_state()) -> process_result().
-process_limit_check(Withdrawal) ->
+-spec process_limit_check(withdrawal_state(), repair_scenario()) -> process_result().
+process_limit_check(Withdrawal, undefined) ->
     WalletID = wallet_id(Withdrawal),
     {ok, Wallet} = get_wallet(WalletID),
     DomainRevision = operation_domain_revision(Withdrawal),
@@ -868,15 +885,33 @@ process_limit_check(Withdrawal) ->
         end,
     {continue, Events}.
 
--spec process_p_transfer_creation(withdrawal_state()) -> process_result().
-process_p_transfer_creation(Withdrawal) ->
+-spec process_p_transfer_creation(withdrawal_state(), repair_scenario()) -> process_result().
+process_p_transfer_creation(Withdrawal, undefined) ->
     FinalCashFlow = make_final_cash_flow(Withdrawal),
     PTransferID = construct_p_transfer_id(Withdrawal),
     {ok, PostingsTransferEvents} = ff_postings_transfer:create(PTransferID, FinalCashFlow),
     {continue, [{p_transfer, Ev} || Ev <- PostingsTransferEvents]}.
 
--spec process_session_creation(withdrawal_state()) -> process_result().
-process_session_creation(Withdrawal) ->
+-spec process_p_transfer_prepare(withdrawal_state(), repair_scenario()) -> process_result().
+process_p_transfer_prepare(Withdrawal, undefined) ->
+    Tr = ff_withdrawal_route_attempt_utils:get_current_p_transfer(attempts(Withdrawal)),
+    {ok, Events} = ff_postings_transfer:prepare(Tr),
+    {continue, [{p_transfer, Ev} || Ev <- Events]}.
+
+-spec process_p_transfer_commit(withdrawal_state(), repair_scenario()) -> process_result().
+process_p_transfer_commit(Withdrawal, undefined) ->
+    Tr = ff_withdrawal_route_attempt_utils:get_current_p_transfer(attempts(Withdrawal)),
+    {ok, Events} = ff_postings_transfer:commit(Tr),
+    {continue, [{p_transfer, Ev} || Ev <- Events]}.
+
+-spec process_p_transfer_cancel(withdrawal_state(), repair_scenario()) -> process_result().
+process_p_transfer_cancel(Withdrawal, undefined) ->
+    Tr = ff_withdrawal_route_attempt_utils:get_current_p_transfer(attempts(Withdrawal)),
+    {ok, Events} = ff_postings_transfer:cancel(Tr),
+    {continue, [{p_transfer, Ev} || Ev <- Events]}.
+
+-spec process_session_creation(withdrawal_state(), repair_scenario()) -> process_result().
+process_session_creation(Withdrawal, undefined) ->
     ID = construct_session_id(Withdrawal),
     #{
         wallet_id := WalletID,
@@ -931,8 +966,8 @@ create_session(ID, TransferData, SessionParams) ->
             ok
     end.
 
--spec process_session_sleep(withdrawal_state()) -> process_result().
-process_session_sleep(Withdrawal) ->
+-spec process_session_sleep(withdrawal_state(), repair_scenario()) -> process_result().
+process_session_sleep(Withdrawal, undefined) ->
     SessionID = session_id(Withdrawal),
     {ok, SessionMachine} = ff_withdrawal_session_machine:get(SessionID),
     Session = ff_withdrawal_session_machine:session(SessionMachine),
@@ -944,8 +979,8 @@ process_session_sleep(Withdrawal) ->
             {continue, [{session_finished, {SessionID, Result}}]}
     end.
 
--spec process_transfer_finish(withdrawal_state()) -> process_result().
-process_transfer_finish(_Withdrawal) ->
+-spec process_transfer_finish(withdrawal_state(), repair_scenario()) -> process_result().
+process_transfer_finish(_Withdrawal, undefined) ->
     {undefined, [{status_changed, succeeded}]}.
 
 -spec process_transfer_fail(fail_type(), withdrawal_state()) -> process_result().
@@ -1587,8 +1622,8 @@ make_change_status_params({failed, _}, {failed, _} = NewStatus, _Withdrawal) ->
         }
     }.
 
--spec process_adjustment(withdrawal_state()) -> process_result().
-process_adjustment(Withdrawal) ->
+-spec process_adjustment(withdrawal_state(), repair_scenario()) -> process_result().
+process_adjustment(Withdrawal, undefined) ->
     #{
         action := Action,
         events := Events0,