From 9aae5a33dffc8543e263908a32937b58a92a56e3 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Tue, 25 Jun 2024 15:31:56 -0300
Subject: [PATCH] feat(ct): add `flaky` test case property

Albeit there's already the `{repeat_until_ok, N}` TC property, it's arguably of reduced
use in a CI environment, as any failure will fail the whole suite, even if the final
execution succeeds.

Here we introduce a new kind of TC property: `{flaky, N}`, where `N` is a positive
integer.

It works similarly to `{repeat_until_ok, N}`: the TC is repeated up to `N` times until it
succeeds or retries are exhausted.  The TC gets the status of its last run: if it
eventually succeeded, it's considered a success.

Example usage:

```erlang
-module(my_SUITE).

all() ->
    %% This test case will be run up to 10 times.
    [{testcase, t_my_flaky_test, [{flaky, 10}]}].

t_my_flaky_test(Config) ->
    K = {?MODULE, ?FUNCTION_NAME},
    N = persistent_term:get(K, 0),
    case N > 5 of
        true ->
            ok;
        false ->
            persistent_term:put(K, N + 1),
            error(boom)
    end.
```

Execution logs:
```
Testing lib.my_SUITE: Stopping test case repeat operation: {flaky,10}
Testing lib.my_SUITE: TEST COMPLETE, 1 ok, 0 failed of 1 test cases
```

Note that, even though it actually ran 6 times, the final statistics are not duplicated.

The produced HTML logs do contain logs from all failures, nevertheless, for debugging
purposes.
---
 lib/common_test/src/ct_framework.erl     |  2 ++
 lib/common_test/src/test_server_ctrl.erl | 31 ++++++++++++++++++++++--
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index dd5ca31a856e..0135e09ec3b1 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -1492,6 +1492,8 @@ report(What,Data) ->
 		    add_to_stats(user_skipped);
 		{_,{auto_skipped,_}} ->
 		    add_to_stats(auto_skipped);
+		{_,{{failed,keep_going}, _}} ->
+		    ok;
 		{_,{SkipOrFail,_Reason}} ->
 		    add_to_stats(SkipOrFail)
 	    end;
diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl
index 569f62eebf32..51791aad1976 100644
--- a/lib/common_test/src/test_server_ctrl.erl
+++ b/lib/common_test/src/test_server_ctrl.erl
@@ -2957,6 +2957,7 @@ run_test_cases_loop([{Mod,Func,Args}=Case|Cases], Config, TimetrapData, Mode0, S
 	    ok
     end,
 
+    maybe_put_flaky_info(Mode0),
     case run_test_case(undefined, Num+1, Mod, Func, Args,
 		       RunInit, TimetrapData, Mode) of
 	%% callback to framework module failed, exit immediately
@@ -3926,6 +3927,8 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit,
                 _ -> not_a_test_fun
             end,
             put(test_server_ok, get(test_server_ok)+1);
+        {_,{failed, keep_going}} ->
+            ok;
         {_,failed} ->
             DiedTime = case Time of
                            died -> case RetVal of
@@ -4172,8 +4175,15 @@ progress(failed, CaseNum, Mod, Func, GrName, Loc, Reason, Time,
     print(major, "=result        failed: ~tp, ~tp", [Reason,LocMaj]),
     print(1, "*** FAILED ~ts ***",
 	  [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]),
+
+    StatusTag = case should_keep_going() of
+        true ->
+            {failed, keep_going};
+        false ->
+            failed
+    end,
     test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName},
-						     {failed,Reason}}]),
+						     {StatusTag,Reason}}]),
     TimeStr = io_lib:format(if is_float(Time) -> "~.3fs";
 			       true -> "~w"
 			    end, [Time]),
@@ -4194,7 +4204,7 @@ progress(failed, CaseNum, Mod, Func, GrName, Loc, Reason, Time,
     print(minor, "~ts",
           ["=== Reason: " ++
            escape_chars(io_lib:format(FStr, [FormattedReason]))]),
-    failed;
+    StatusTag;
 
 progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time,
 	 Comment0, {St0,St1}) ->
@@ -5839,6 +5849,10 @@ do_update_repeat_data(ok,{repeat_until_ok=RT,M,N}) ->
     report_repeat_testcase(M,N),
     report_stop_repeat_testcase(RT,{RT,N}),
     false;
+do_update_repeat_data(ok,{flaky=RT,M,N}) ->
+    report_repeat_testcase(M,N),
+    report_stop_repeat_testcase(RT,{RT,N}),
+    false;
 do_update_repeat_data(failed,{repeat_until_fail=RT,M,N}) ->
     report_repeat_testcase(M,N),
     report_stop_repeat_testcase(RT,{RT,N}),
@@ -5858,3 +5872,16 @@ report_repeat_testcase(M,forever) ->
     print(minor, "~n=== Repeated test case: ~w of infinity", [M]);
 report_repeat_testcase(M,N) ->
     print(minor, "~n=== Repeated test case: ~w of ~w", [M,N]).
+
+maybe_put_flaky_info([{_Ref, [{repeat, {flaky, N, M}}], _Time} | _]) ->
+    put('$ct_flaky_info', {N, M});
+maybe_put_flaky_info(_) ->
+    erase('$ct_flaky_info').
+
+should_keep_going() ->
+    case get('$ct_flaky_info') of
+        {N, M} when N < M ->
+            true;
+        _ ->
+            false
+    end.