Use Common Test for testing when Gradualizer should pass, fail, and its known problems

Makefile

@@ -149,6 +149,9 @@ eunit: compile-tests
 	erl $(ERL_OPTS) -noinput -pa ebin -pa test -eval \
 	 '$(erl_run_eunit), halt().'
 
+ct:
+	@rebar3 ct --label "git: $$(git describe --tags --always) $$(git diff --no-ext-diff --quiet --exit-code || echo '(modified)')"
+
 cli-tests: bin/gradualizer test/arg.beam
 	# CLI test cases
 	# 1. When checking a dir with erl files, erl file names are printed

rebar.config

@@ -4,10 +4,7 @@
           {deps,
            [
             {proper, {git, "https://github.com/proper-testing/proper.git", {branch, "master"}}}
-           ]},
-          %% see the maybe expression fail;
-          %% the VM also needs to be configured to load the module
-          {erl_opts, [{feature,maybe_expr,enable}]}
+           ]}
          ]}
  ]}.
 

src/typechecker.erl

@@ -5746,8 +5746,10 @@ type_check_forms(Forms, Opts) ->
 %% a Gradualizer (NOT the checked program!) error.
 -spec type_check_form_with_timeout(expr(), [any()], boolean(), env(), [any()]) -> [any()].
 type_check_form_with_timeout(Function, Errors, StopOnFirstError, Env, Opts) ->
-    %% TODO: make FormCheckTimeOut configurable
-    FormCheckTimeOut = ?form_check_timeout_ms,
+    FormCheckTimeOut = case lists:keyfind(form_check_timeout_ms, 1, Opts) of
+                           false -> ?form_check_timeout_ms;
+                           {form_check_timeout_ms, MS} -> MS
+                       end,
     ?verbose(Env, "Spawning async task...~n", []),
     Self = self(),
     Task = fun () ->

test/gradualizer_dynamic_suite.erl

@@ -0,0 +1,50 @@
+-module(gradualizer_dynamic_suite).
+
+-export([reload/1]).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("stdlib/include/assert.hrl").
+
+reload(Config) ->
+    Module = ?config(dynamic_suite_module, Config),
+    Path = ?config(dynamic_suite_test_path, Config),
+    ?assert(Module /= undefined),
+    ?assert(Path /= undefined),
+    Forms = get_forms(Module),
+    FilesForms = map_erl_files(fun (File) ->
+                                       make_test_form(Forms, File, Config)
+                               end, Path),
+    {TestFiles, TestForms} = lists:unzip(FilesForms),
+    TestNames = [ list_to_atom(filename:basename(File, ".erl")) || File <- TestFiles ],
+    ct:pal("All tests found under ~s:\n~p\n", [Path, TestNames]),
+    NewForms = Forms ++ TestForms ++ [{eof, 0}],
+    {ok, _} = merl:compile_and_load(NewForms),
+    {ok, TestNames}.
+
+map_erl_files(Fun, Dir) ->
+    Files = filelib:wildcard(filename:join(Dir, "*.erl")),
+    [{filename:basename(File), Fun(File)} || File <- Files].
+
+make_test_form(Forms, File, Config) ->
+    TestTemplateName = ?config(dynamic_test_template, Config),
+    ?assert(TestTemplateName /= undefined),
+    TestTemplate = merl:quote("'@Name'(_) -> _@Body."),
+    {function, _Anno, _Name, 1, Clauses} = lists:keyfind(TestTemplateName, 3, Forms),
+    [{clause, _, _Args, _Guards, ClauseBodyTemplate}] = Clauses,
+    TestName = filename:basename(File, ".erl"),
+    ClauseBody = merl:subst(ClauseBodyTemplate, [{'File', erl_syntax:string(File)}]),
+    TestEnv = [
+               {'Name', erl_syntax:atom(TestName)},
+               {'Body', ClauseBody}
+              ],
+    erl_syntax:revert(merl:subst(TestTemplate, TestEnv)).
+
+get_forms(Module) ->
+    ModPath = code:which(Module),
+    {ok, {Module, [Abst]}} = beam_lib:chunks(ModPath, [abstract_code]),
+    {abstract_code, {raw_abstract_v1, Forms}} = Abst,
+    StripEnd = fun
+                   ({eof, _}) -> false;
+                   (_) -> true
+               end,
+    lists:filter(StripEnd, Forms).

test/known_problems_should_fail_SUITE.erl

@@ -0,0 +1,90 @@
+-module(known_problems_should_fail_SUITE).
+
+-compile([export_all, nowarn_export_all]).
+
+%% EUnit has some handy macros, so let's use it, too
+-include_lib("eunit/include/eunit.hrl").
+
+%% Test server callbacks
+-export([suite/0,
+         all/0,
+         groups/0,
+         init_per_suite/1, end_per_suite/1,
+         init_per_group/2, end_per_group/2,
+         init_per_testcase/2, end_per_testcase/2]).
+
+suite() ->
+    [{timetrap, {minutes, 10}}].
+
+init_per_suite(Config0) ->
+    AppBase = code:lib_dir(gradualizer),
+    Config = [
+              {dynamic_suite_module, ?MODULE},
+              {dynamic_suite_test_path, filename:join(AppBase, "test/known_problems/should_fail")},
+              {dynamic_test_template, known_problems_should_fail_template}
+             ] ++ Config0,
+    {ok, _} = application:ensure_all_started(gradualizer),
+    ok = load_prerequisites(AppBase),
+    {ok, TestNames} = gradualizer_dynamic_suite:reload(Config),
+    case all_tests() of
+        TestNames -> ok;
+        _ -> ct:fail("Please update all_tests/0 to list all tests")
+    end,
+    Config.
+
+load_prerequisites(AppBase) ->
+    %% exhaustive_user_type.erl is referenced by exhaustive_remote_user_type.erl
+    gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_fail/exhaustive_user_type.erl")]),
+    ok.
+
+end_per_suite(_Config) ->
+    ok = application:stop(gradualizer),
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, _Config) ->
+    ok.
+
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+end_per_testcase(_TestCase, _Config) ->
+    ok.
+
+all() ->
+    [{group, all_tests}].
+
+groups() ->
+    [{all_tests, [parallel], all_tests()}].
+
+all_tests() ->
+    [arith_op,binary_comprehension,case_pattern_should_fail,
+     exhaustive_argumentwise,exhaustive_expr,exhaustive_map_variants,
+     exhaustive_remote_map_variants,guard_should_fail,infer_any_pattern,
+     intersection_with_any_should_fail,intersection_with_unreachable,
+     lambda_wrong_args,map_refinement_fancy,poly_lists_map_should_fail,
+     poly_should_fail,recursive_types_should_fail,refine_ty_vars,sample].
+
+known_problems_should_fail_template(_@File) ->
+    Result = safe_type_check_file(_@File, [return_errors]),
+    case Result of
+        crash ->
+            ok;
+        Errors ->
+            ErrorsExceptTimeouts = lists:filter(
+                                     fun ({_File, {form_check_timeout, _}}) -> false; (_) -> true end,
+                                     Errors),
+            ?assertEqual(0, length(ErrorsExceptTimeouts))
+    end.
+
+safe_type_check_file(File) ->
+    safe_type_check_file(File, []).
+
+safe_type_check_file(File, Opts) ->
+    try
+        gradualizer:type_check_file(File, Opts)
+    catch
+        _:_ -> crash
+    end.

test/known_problems_should_pass_SUITE.erl

@@ -0,0 +1,85 @@
+-module(known_problems_should_pass_SUITE).
+
+-compile([export_all, nowarn_export_all]).
+
+%% EUnit has some handy macros, so let's use it, too
+-include_lib("eunit/include/eunit.hrl").
+
+%% Test server callbacks
+-export([suite/0,
+         all/0,
+         groups/0,
+         init_per_suite/1, end_per_suite/1,
+         init_per_group/2, end_per_group/2,
+         init_per_testcase/2, end_per_testcase/2]).
+
+suite() ->
+    [{timetrap, {minutes, 10}}].
+
+init_per_suite(Config0) ->
+    AppBase = code:lib_dir(gradualizer),
+    Config = [
+              {dynamic_suite_module, ?MODULE},
+              {dynamic_suite_test_path, filename:join(AppBase, "test/known_problems/should_pass")},
+              {dynamic_test_template, known_problems_should_pass_template}
+             ] ++ Config0,
+    {ok, _} = application:ensure_all_started(gradualizer),
+    ok = load_prerequisites(AppBase),
+    {ok, TestNames} = gradualizer_dynamic_suite:reload(Config),
+    case all_tests() of
+        TestNames -> ok;
+        _ -> ct:fail("Please update all_tests/0 to list all tests")
+    end,
+    Config.
+
+load_prerequisites(_AppBase) ->
+    ok.
+
+end_per_suite(_Config) ->
+    ok = application:stop(gradualizer),
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, _Config) ->
+    ok.
+
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+end_per_testcase(_TestCase, _Config) ->
+    ok.
+
+all() ->
+    [{group, all_tests}].
+
+groups() ->
+    [{all_tests, [parallel], all_tests()}].
+
+all_tests() ->
+    [arith_op_arg_types,binary_exhaustiveness_checking_should_pass,
+     call_intersection_function_with_union_arg_should_pass,
+     different_normalization_levels,elixir_list_first,error_in_guard,
+     fun_subtyping,generator_var_shadow,inner_union_subtype_of_root_union,
+     intersection_should_pass,intersection_with_any,list_concat_op_should_pass,
+     list_tail,map_pattern_duplicate_key,maybe_expr,poly_should_pass,
+     poly_type_vars,recursive_types,refine_bound_var_on_mismatch,
+     refine_bound_var_with_guard_should_pass,refine_comparison_should_pass,
+     refine_list_tail,union_fun].
+
+known_problems_should_pass_template(_@File) ->
+    {ok, Forms} = gradualizer_file_utils:get_forms_from_erl(_@File, []),
+    ExpectedErrors = typechecker:number_of_exported_functions(Forms),
+    ReturnedErrors = length(safe_type_check_file(_@File, [return_errors])),
+    ?assertEqual(ExpectedErrors, ReturnedErrors).
+
+safe_type_check_file(File) ->
+    safe_type_check_file(File, []).
+
+safe_type_check_file(File, Opts) ->
+    try
+        gradualizer:type_check_file(File, Opts)
+    catch
+        _:_ -> crash
+    end.

test/should_fail_SUITE.erl

@@ -0,0 +1,111 @@
+-module(should_fail_SUITE).
+
+-compile([export_all, nowarn_export_all]).
+
+%% EUnit has some handy macros, so let's use it, too
+-include_lib("eunit/include/eunit.hrl").
+
+%% Test server callbacks
+-export([suite/0,
+         all/0,
+         groups/0,
+         init_per_suite/1, end_per_suite/1,
+         init_per_group/2, end_per_group/2,
+         init_per_testcase/2, end_per_testcase/2]).
+
+suite() ->
+    [{timetrap, {minutes, 10}}].
+
+init_per_suite(Config0) ->
+    AppBase = code:lib_dir(gradualizer),
+    Config = [
+              {dynamic_suite_module, ?MODULE},
+              {dynamic_suite_test_path, filename:join(AppBase, "test/should_fail")},
+              {dynamic_test_template, should_fail_template}
+             ] ++ Config0,
+    {ok, _} = application:ensure_all_started(gradualizer),
+    ok = load_prerequisites(AppBase),
+    {ok, TestNames} = gradualizer_dynamic_suite:reload(Config),
+    case all_tests() of
+        TestNames -> ok;
+        _ -> ct:fail("Please update all_tests/0 to list all tests")
+    end,
+    Config.
+
+load_prerequisites(AppBase) ->
+    %% user_types.erl is referenced by opaque_fail.erl.
+    %% It is not in the sourcemap of the DB so let's import it manually
+    gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_pass/user_types.erl")]),
+    %% exhaustive_user_type.erl is referenced by exhaustive_remote_user_type.erl
+    gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_fail/exhaustive_user_type.erl")]),
+    ok.
+
+end_per_suite(_Config) ->
+    ok = application:stop(gradualizer),
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, _Config) ->
+    ok.
+
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+end_per_testcase(_TestCase, _Config) ->
+    ok.
+
+all() ->
+    [{group, all_tests}].
+
+groups() ->
+    [{all_tests, [parallel], all_tests()}].
+
+all_tests() ->
+    [annotated_types_fail,arg,arith_op_fail,arity_mismatch,
+     bc_fail,bin_expression,bin_type_error,branch,branch2,call,
+     call_intersection_function_with_union_arg_fail,case_pattern,
+     case_pattern2,catch_expr_fail,cons,covariant_map_keys_fail,
+     cyclic_type_vars,depth,exhaustive,exhaustive_float,
+     exhaustive_list_variants,exhaustive_refinable_map_variants,
+     exhaustive_remote_user_type,exhaustive_string_variants,
+     exhaustive_type,exhaustive_user_type,
+     exhaustiveness_check_toggling,generator,guard_fail,
+     imported_undef,infer_enabled,intersection_check,
+     intersection_fail,intersection_infer,
+     intersection_with_any_fail,iodata_fail,lambda_not_fun,
+     lc_generator_not_none_fail,lc_not_list,list_infer_fail,
+     list_op,list_op_should_fail,list_union_fail,
+     lists_map_nonempty_fail,literal_char,literal_patterns,
+     logic_op,map_entry,map_fail,map_failing_expr,
+     map_failing_subtyping,map_field_invalid_update,map_literal,
+     map_pattern_fail,map_refinement_fail,map_type_error,match,
+     messaging_fail,module_info_fail,named_fun_fail,
+     named_fun_infer_fail,nil,no_idempotent_xor,
+     non_neg_plus_pos_is_pos_fail,
+     nonempty_list_match_in_head_nonexhaustive,
+     nonempty_string_fail,opaque_fail,operator_pattern_fail,
+     pattern,pattern_record_fail,poly_fail,poly_lists_map_fail,
+     poly_union_lower_bound_fail,pp_intersection,record,
+     record_exhaustive,record_field,record_index,
+     record_info_fail,record_refinement_fail,record_update,
+     record_wildcard_fail,recursive_type_fail,
+     recursive_types_failing,rel_op,return_fun_fail,
+     rigid_type_variables_fail,send_fail,shortcut_ops_fail,
+     spec_and_fun_clause_intersection_fail,string_literal,
+     tuple_union_arg_fail,tuple_union_fail,tuple_union_pattern,
+     tuple_union_refinement,type_refinement_fail,unary_op,
+     unary_plus_fail,union_with_any,unreachable_after_refinement].
+
+should_fail_template(_@File) ->
+    Errors = gradualizer:type_check_file(_@File, [return_errors]),
+    Timeouts = [ E || {_File, {form_check_timeout, _}} = E <- Errors],
+    ?assertEqual(0, length(Timeouts)),
+    %% Test that error formatting doesn't crash
+    Opts = [{fmt_location, brief},
+            {fmt_expr_fun, fun erl_prettypr:format/1}],
+    lists:foreach(fun({_, Error}) -> gradualizer_fmt:handle_type_error(Error, Opts) end, Errors),
+    {ok, Forms} = gradualizer_file_utils:get_forms_from_erl(_@File, []),
+    ExpectedErrors = typechecker:number_of_exported_functions(Forms),
+    ?assertEqual(ExpectedErrors, length(Errors)).

test/should_pass/module_info.erl → test/should_pass/module_info_pass.erl

@@ -1,4 +1,4 @@
--module(module_info).
+-module(module_info_pass).
 
 -compile([export_all, nowarn_export_all]).
 
@@ -18,4 +18,4 @@ unary_direct() ->
 -spec unary_var() -> atom().
 unary_var() ->
     I = erlang:module_info(module),
-    I.
+    I.