Update summarise-ct-results

.github/workflows/ci.yml

@@ -25,3 +25,5 @@ jobs:
       run: rebar3 ct
     - name: Check docs
       run: rebar3 ex_doc
+    - name: Check format
+      run: rebar3 fmt -c

README.md

@@ -5,13 +5,38 @@ It writes a `groups.summary` file into the suite log directory.
 
 Used in [MongooseIM big tests](https://github.com/esl/MongooseIM/tree/master/big_tests).
 
-See [summarise-ct-results script](https://github.com/esl/MongooseIM/blob/master/tools/summarise-ct-results), which
-reads the `groups.summary` file.
-
 This repo contains the hook and tests for it.
 
 [Hex package](https://hex.pm/packages/ct_groups_summary_hook)
 
 ## Credits
 
 `ct_groups_summary_hook` was originally created within [the MongooseIM repository as part of its test framework](https://github.com/esl/MongooseIM).
+
+# Test Execution Summary
+
+See [summarise-ct-results script](priv/summarise-ct-results), which
+reads the `groups.summary` file.
+
+Usage example (takes a list of directories as arguments):
+
+```
+priv/summarise-ct-results _build/test/logs/last/
+CT results:
+    1 groups passed
+    0 groups failed
+    0 tests eventually passed
+    0 tests with end_per_testcase failed
+    1 tests passed
+    0 tests failed
+    0 tests skipped by user
+    0 tests skipped automatically
+```
+
+Returns with the exit code 0 on the test execution success.
+
+# Apply code formatter before commit
+
+```
+rebar3 fmt -w
+```

priv/summarise-ct-results

@@ -0,0 +1,165 @@
+#!/usr/bin/env escript
+
+main(Directories) ->
+    {OkTests, FailedTests, UserSkipped, AutoSkipped} =
+        add_up_suite_results(suite_summaries(Directories)),
+    {OkGroups, FailedGroups, EventuallyOkTests, EndPerTestcaseFailed} =
+        add_up_group_results(group_summaries(Directories)),
+    verify_results(
+        OkGroups,
+        FailedGroups,
+        EventuallyOkTests,
+        EndPerTestcaseFailed,
+        OkTests,
+        FailedTests,
+        UserSkipped,
+        AutoSkipped
+    ).
+
+verify_results(
+    0 = _OkGroups,
+    0 = _FailedGroups,
+    0 = _EventuallyOkTests,
+    0 = _EndPerTestcaseFailed,
+    OkTests,
+    FailedTests,
+    UserSkipped,
+    AutoSkipped
+) ->
+    %% These are likely small tests - we're not writing
+    %% groups.summaries files, therefore we get 0 for both group kinds.
+    verify_results_no_groups(OkTests, FailedTests, UserSkipped, AutoSkipped);
+verify_results(
+    OkGroups,
+    FailedGroups,
+    EventuallyOkTests,
+    EndPerTestcaseFailed,
+    OkTests,
+    FailedTests,
+    UserSkipped,
+    AutoSkipped
+) ->
+    verify_results_with_groups(
+        OkGroups,
+        FailedGroups,
+        EventuallyOkTests,
+        EndPerTestcaseFailed,
+        OkTests,
+        FailedTests,
+        UserSkipped,
+        AutoSkipped
+    ).
+
+verify_results_no_groups(OkTests, FailedTests, UserSkipped, AutoSkipped) ->
+    io:format(
+        "CT results:~n"
+        "    ~p tests passed~n"
+        "    ~p tests failed~n"
+        "    ~p tests skipped by user~n"
+        "    ~p tests skipped automatically~n",
+        [OkTests, FailedTests, UserSkipped, AutoSkipped]
+    ),
+    case OkTests =< 0 of
+        true ->
+            erlang:halt(100404);
+        _ ->
+            erlang:halt(FailedTests + AutoSkipped)
+    end.
+
+verify_results_with_groups(
+    OkGroups,
+    FailedGroups,
+    EventuallyOkTests,
+    EndPerTestcaseFailed,
+    OkTests,
+    FailedTests,
+    UserSkipped,
+    AutoSkipped
+) ->
+    io:format(
+        "CT results:~n"
+        "    ~p groups passed~n"
+        "    ~p groups failed~n"
+        "    ~p tests eventually passed~n"
+        "    ~p tests with end_per_testcase failed~n"
+        "    ~p tests passed~n"
+        "    ~p tests failed~n"
+        "    ~p tests skipped by user~n"
+        "    ~p tests skipped automatically~n",
+        [
+            OkGroups,
+            FailedGroups,
+            EventuallyOkTests,
+            EndPerTestcaseFailed,
+            OkTests,
+            FailedTests,
+            UserSkipped,
+            AutoSkipped
+        ]
+    ),
+    Code = FailedGroups + FailedTests - EventuallyOkTests + EndPerTestcaseFailed + AutoSkipped,
+    print_if(OkTests =:= 0, "No tests were executed successfully~n"),
+    print_if(OkGroups =:= 0, "No test groups were executed successfully~n"),
+    %% this can give false negative result for eventually passing
+    %% groups with 'sequence' and 'repeat_until_all_ok' flags, but
+    %% at least it's not false positive result.
+    print_if(AutoSkipped > 0, "Failing the test due to auto skipped cases~n"),
+    print_if(FailedGroups > 0, "Failing the test due to failed rerun groups~n"),
+    print_if(FailedTests > EventuallyOkTests, "Failing the test due to failed test cases~n"),
+    print_if(EndPerTestcaseFailed > 0, "Failing the test due to failed end_per_testcase~n"),
+    if
+        Code =:= 0, OkTests =:= 0 ->
+            erlang:halt(1);
+        Code =:= 0, OkGroups =:= 0 ->
+            erlang:halt(1);
+        true ->
+            erlang:halt(Code)
+    end.
+
+print_if(true, Str) ->
+    io:format(Str);
+print_if(false, _Str) ->
+    ok.
+
+suite_summaries(Directories) ->
+    summaries(Directories, "suite.summary").
+
+group_summaries(Directories) ->
+    summaries(Directories, "groups.summary").
+
+summaries(Directories, Filename) ->
+    lists:foldl(
+        fun(Dir, Acc) ->
+            Wildcard = filename:join([Dir, "*.logs", "*", Filename]),
+            Acc ++ filelib:wildcard(Wildcard)
+        end,
+        [],
+        Directories
+    ).
+
+add_up_suite_results(SuiteSummaries) ->
+    lists:foldl(
+        fun(Filename, {OkAcc, FAcc, USAcc, ASAcc}) ->
+            {ok, [{summary, Summary}]} = file:consult(Filename),
+            [Ok, F, US, AS | _] = tuple_to_list(Summary),
+            {OkAcc + Ok, FAcc + F, USAcc + US, ASAcc + AS}
+        end,
+        {0, 0, 0, 0},
+        SuiteSummaries
+    ).
+
+add_up_group_results(GroupSummaries) ->
+    lists:foldl(
+        fun(Filename, {OkAcc, FAcc, EvOkAcc, EndPerTestcaseFailedAcc}) ->
+            {ok, Terms} = file:consult(Filename),
+            #{
+                eventually_ok_tests := EvOk,
+                end_per_testcase_failures := EndPerTestcaseFailed,
+                groups_summary := {Ok, F}
+            } =
+                maps:from_list(Terms),
+            {OkAcc + Ok, FAcc + F, EvOkAcc + EvOk, EndPerTestcaseFailedAcc + EndPerTestcaseFailed}
+        end,
+        {0, 0, 0, 0},
+        GroupSummaries
+    ).

rebar.config

@@ -1,4 +1,4 @@
-{plugins, [rebar3_ex_doc, rebar3_hex]}.
+{plugins, [rebar3_ex_doc, rebar3_hex, erlfmt]}.
 
 {ex_doc, [
     {source_url, <<"https://github.com/esl/ct_groups_summary_hook">>},
@@ -7,3 +7,12 @@
 ]}.
 
 {hex, [{doc, ex_doc}]}.
+
+{erlfmt, [
+    {files, [
+        "{src,test}/*.{erl,app.src}",
+        "ct_app/{src,test}/*.{erl,app.src}",
+        "rebar.config",
+        "priv/summarise-ct-results"
+    ]}
+]}.

src/ct_groups_summary_hook.app.src

@@ -1,14 +1,13 @@
-{application, ct_groups_summary_hook,
- [
-  {description, "Common Tests Group Summary Hook"},
-  {vsn, git},
-  {registered, []},
-  {applications, [
-                  kernel,
-                  stdlib
-                 ]},
-  {maintainers, ["Erlang Solutions"]},
-  {licenses, ["Apache-2.0"]},
-  {links, [{"GitHub", "https://github.com/esl/ct_groups_summary_hook/"}]},
-  {env, []}
- ]}.
+{application, ct_groups_summary_hook, [
+    {description, "Common Tests Group Summary Hook"},
+    {vsn, git},
+    {registered, []},
+    {applications, [
+        kernel,
+        stdlib
+    ]},
+    {maintainers, ["Erlang Solutions"]},
+    {licenses, ["Apache-2.0"]},
+    {links, [{"GitHub", "https://github.com/esl/ct_groups_summary_hook/"}]},
+    {env, []}
+]}.