diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index dd5ca31a856e..0135e09ec3b1 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -1492,6 +1492,8 @@ report(What,Data) -> add_to_stats(user_skipped); {_,{auto_skipped,_}} -> add_to_stats(auto_skipped); + {_,{{failed,keep_going}, _}} -> + ok; {_,{SkipOrFail,_Reason}} -> add_to_stats(SkipOrFail) end; diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl index 569f62eebf32..51791aad1976 100644 --- a/lib/common_test/src/test_server_ctrl.erl +++ b/lib/common_test/src/test_server_ctrl.erl @@ -2957,6 +2957,7 @@ run_test_cases_loop([{Mod,Func,Args}=Case|Cases], Config, TimetrapData, Mode0, S ok end, + maybe_put_flaky_info(Mode0), case run_test_case(undefined, Num+1, Mod, Func, Args, RunInit, TimetrapData, Mode) of %% callback to framework module failed, exit immediately @@ -3926,6 +3927,8 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, _ -> not_a_test_fun end, put(test_server_ok, get(test_server_ok)+1); + {_,{failed, keep_going}} -> + ok; {_,failed} -> DiedTime = case Time of died -> case RetVal of @@ -4172,8 +4175,15 @@ progress(failed, CaseNum, Mod, Func, GrName, Loc, Reason, Time, print(major, "=result failed: ~tp, ~tp", [Reason,LocMaj]), print(1, "*** FAILED ~ts ***", [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), + + StatusTag = case should_keep_going() of + true -> + {failed, keep_going}; + false -> + failed + end, test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName}, - {failed,Reason}}]), + {StatusTag,Reason}}]), TimeStr = io_lib:format(if is_float(Time) -> "~.3fs"; true -> "~w" end, [Time]), @@ -4194,7 +4204,7 @@ progress(failed, CaseNum, Mod, Func, GrName, Loc, Reason, Time, print(minor, "~ts", ["=== Reason: " ++ escape_chars(io_lib:format(FStr, [FormattedReason]))]), - failed; + StatusTag; progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time, Comment0, {St0,St1}) -> @@ -5839,6 +5849,10 @@ do_update_repeat_data(ok,{repeat_until_ok=RT,M,N}) -> report_repeat_testcase(M,N), report_stop_repeat_testcase(RT,{RT,N}), false; +do_update_repeat_data(ok,{flaky=RT,M,N}) -> + report_repeat_testcase(M,N), + report_stop_repeat_testcase(RT,{RT,N}), + false; do_update_repeat_data(failed,{repeat_until_fail=RT,M,N}) -> report_repeat_testcase(M,N), report_stop_repeat_testcase(RT,{RT,N}), @@ -5858,3 +5872,16 @@ report_repeat_testcase(M,forever) -> print(minor, "~n=== Repeated test case: ~w of infinity", [M]); report_repeat_testcase(M,N) -> print(minor, "~n=== Repeated test case: ~w of ~w", [M,N]). + +maybe_put_flaky_info([{_Ref, [{repeat, {flaky, N, M}}], _Time} | _]) -> + put('$ct_flaky_info', {N, M}); +maybe_put_flaky_info(_) -> + erase('$ct_flaky_info'). + +should_keep_going() -> + case get('$ct_flaky_info') of + {N, M} when N < M -> + true; + _ -> + false + end.