Skip to content

Commit

Permalink
feat(ct): add flaky test case property
Browse files Browse the repository at this point in the history
Albeit there's already the `{repeat_until_ok, N}` TC property, it's arguably of reduced
use in a CI environment, as any failure will fail the whole suite, even if the final
execution succeeds.

Here we introduce a new kind of TC property: `{flaky, N}`, where `N` is a positive
integer.

It works similarly to `{repeat_until_ok, N}`: the TC is repeated up to `N` times until it
succeeds or retries are exhausted.  The TC gets the status of its last run: if it
eventually succeeded, it's considered a success.

Example usage:

```erlang
-module(my_SUITE).

all() ->
    %% This test case will be run up to 10 times.
    [{testcase, t_my_flaky_test, [{flaky, 10}]}].

t_my_flaky_test(Config) ->
    K = {?MODULE, ?FUNCTION_NAME},
    N = persistent_term:get(K, 0),
    case N > 5 of
        true ->
            ok;
        false ->
            persistent_term:put(K, N + 1),
            error(boom)
    end.
```

Execution logs:
```
Testing lib.my_SUITE: Stopping test case repeat operation: {flaky,10}
Testing lib.my_SUITE: TEST COMPLETE, 1 ok, 0 failed of 1 test cases
```

Note that, even though it actually ran 6 times, the final statistics are not duplicated.

The produced HTML logs do contain logs from all failures, nevertheless, for debugging
purposes.
  • Loading branch information
thalesmg committed Jun 25, 2024
1 parent 8cea620 commit 9aae5a3
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 2 deletions.
2 changes: 2 additions & 0 deletions lib/common_test/src/ct_framework.erl
Original file line number Diff line number Diff line change
Expand Up @@ -1492,6 +1492,8 @@ report(What,Data) ->
add_to_stats(user_skipped);
{_,{auto_skipped,_}} ->
add_to_stats(auto_skipped);
{_,{{failed,keep_going}, _}} ->
ok;
{_,{SkipOrFail,_Reason}} ->
add_to_stats(SkipOrFail)
end;
Expand Down
31 changes: 29 additions & 2 deletions lib/common_test/src/test_server_ctrl.erl
Original file line number Diff line number Diff line change
Expand Up @@ -2957,6 +2957,7 @@ run_test_cases_loop([{Mod,Func,Args}=Case|Cases], Config, TimetrapData, Mode0, S
ok
end,

maybe_put_flaky_info(Mode0),
case run_test_case(undefined, Num+1, Mod, Func, Args,
RunInit, TimetrapData, Mode) of
%% callback to framework module failed, exit immediately
Expand Down Expand Up @@ -3926,6 +3927,8 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit,
_ -> not_a_test_fun
end,
put(test_server_ok, get(test_server_ok)+1);
{_,{failed, keep_going}} ->
ok;
{_,failed} ->
DiedTime = case Time of
died -> case RetVal of
Expand Down Expand Up @@ -4172,8 +4175,15 @@ progress(failed, CaseNum, Mod, Func, GrName, Loc, Reason, Time,
print(major, "=result failed: ~tp, ~tp", [Reason,LocMaj]),
print(1, "*** FAILED ~ts ***",
[get_info_str(Mod,Func, CaseNum, get(test_server_cases))]),

StatusTag = case should_keep_going() of
true ->
{failed, keep_going};
false ->
failed
end,
test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName},
{failed,Reason}}]),
{StatusTag,Reason}}]),
TimeStr = io_lib:format(if is_float(Time) -> "~.3fs";
true -> "~w"
end, [Time]),
Expand All @@ -4194,7 +4204,7 @@ progress(failed, CaseNum, Mod, Func, GrName, Loc, Reason, Time,
print(minor, "~ts",
["=== Reason: " ++
escape_chars(io_lib:format(FStr, [FormattedReason]))]),
failed;
StatusTag;

progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time,
Comment0, {St0,St1}) ->
Expand Down Expand Up @@ -5839,6 +5849,10 @@ do_update_repeat_data(ok,{repeat_until_ok=RT,M,N}) ->
report_repeat_testcase(M,N),
report_stop_repeat_testcase(RT,{RT,N}),
false;
do_update_repeat_data(ok,{flaky=RT,M,N}) ->
report_repeat_testcase(M,N),
report_stop_repeat_testcase(RT,{RT,N}),
false;
do_update_repeat_data(failed,{repeat_until_fail=RT,M,N}) ->
report_repeat_testcase(M,N),
report_stop_repeat_testcase(RT,{RT,N}),
Expand All @@ -5858,3 +5872,16 @@ report_repeat_testcase(M,forever) ->
print(minor, "~n=== Repeated test case: ~w of infinity", [M]);
report_repeat_testcase(M,N) ->
print(minor, "~n=== Repeated test case: ~w of ~w", [M,N]).

maybe_put_flaky_info([{_Ref, [{repeat, {flaky, N, M}}], _Time} | _]) ->
put('$ct_flaky_info', {N, M});
maybe_put_flaky_info(_) ->
erase('$ct_flaky_info').

should_keep_going() ->
case get('$ct_flaky_info') of
{N, M} when N < M ->
true;
_ ->
false
end.

0 comments on commit 9aae5a3

Please sign in to comment.