From ac468056f954ab84679af35f80128ba1ad308834 Mon Sep 17 00:00:00 2001 From: Xinhao Yuan Date: Thu, 23 Jan 2025 08:11:15 -0800 Subject: [PATCH] No public description PiperOrigin-RevId: 718872113 --- centipede/BUILD | 2 + centipede/centipede.cc | 4 +- centipede/centipede_callbacks.cc | 5 + centipede/centipede_interface.cc | 316 +++++++++--------- centipede/environment.cc | 5 +- centipede/environment.h | 3 + centipede/environment_test.cc | 2 +- common/BUILD | 12 + common/CMakeLists.txt | 14 + common/bazel.cc | 120 +++++++ common/bazel.h | 44 +++ e2e_tests/BUILD | 2 + e2e_tests/corpus_database_test.cc | 247 ++++++++++---- e2e_tests/functional_test.cc | 179 +++++----- fuzztest/BUILD | 5 +- fuzztest/CMakeLists.txt | 1 + fuzztest/init_fuzztest.cc | 113 +++++-- fuzztest/internal/centipede_adaptor.cc | 414 +++++++++++++----------- fuzztest/internal/centipede_adaptor.h | 3 + fuzztest/internal/configuration.cc | 25 +- fuzztest/internal/configuration.h | 19 +- fuzztest/internal/configuration_test.cc | 15 +- fuzztest/internal/runtime.cc | 25 ++ fuzztest/internal/runtime.h | 9 +- fuzztest/internal/runtime_test.cc | 4 +- 25 files changed, 1065 insertions(+), 523 deletions(-) create mode 100644 common/bazel.cc create mode 100644 common/bazel.h diff --git a/centipede/BUILD b/centipede/BUILD index 3dea36d6..668c0d5e 100644 --- a/centipede/BUILD +++ b/centipede/BUILD @@ -824,6 +824,7 @@ cc_library( ":util", ":workdir", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/cleanup", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/log", "@com_google_absl//absl/log:check", @@ -833,6 +834,7 @@ cc_library( "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/time", "@com_google_absl//absl/types:span", + "@com_google_fuzztest//common:bazel", "@com_google_fuzztest//common:blob_file", "@com_google_fuzztest//common:defs", "@com_google_fuzztest//common:hash", diff --git a/centipede/centipede.cc b/centipede/centipede.cc index 492e5b71..908f95a8 100644 --- a/centipede/centipede.cc +++ b/centipede/centipede.cc @@ -811,7 +811,9 @@ void Centipede::ReportCrash(std::string_view binary, const std::vector &input_vec, const BatchResult &batch_result) { CHECK_EQ(input_vec.size(), batch_result.results().size()); - if (ShouldStop()) return; + // Skip reporting only if RequestEarlyStop is called with a failure exit code. + // Still report if time runs out. + if (ShouldStop() && ExitCode() != 0) return; if (++num_crashes_ > env_.max_num_crash_reports) return; diff --git a/centipede/centipede_callbacks.cc b/centipede/centipede_callbacks.cc index 8b36a140..202965e0 100644 --- a/centipede/centipede_callbacks.cc +++ b/centipede/centipede_callbacks.cc @@ -264,6 +264,11 @@ bool CentipedeCallbacks::GetSeedsViaExternalBinary( .temp_file_path = temp_input_file_path_}}; const int retval = cmd.Execute(); + if (env_.print_runner_log) { + LOG(INFO) << "Getting seeds via external binary returns " << retval; + PrintExecutionLog(); + } + std::vector seed_input_filenames; for (const auto &dir_ent : std::filesystem::directory_iterator(output_dir)) { seed_input_filenames.push_back(dir_ent.path().filename()); diff --git a/centipede/centipede_interface.cc b/centipede/centipede_interface.cc index f9d02525..21e5e8c9 100644 --- a/centipede/centipede_interface.cc +++ b/centipede/centipede_interface.cc @@ -31,6 +31,7 @@ #include #include "absl/base/optimization.h" +#include "absl/cleanup/cleanup.h" #include "absl/container/flat_hash_set.h" #include "absl/log/check.h" #include "absl/log/log.h" @@ -63,6 +64,7 @@ #include "./centipede/thread_pool.h" #include "./centipede/util.h" #include "./centipede/workdir.h" +#include "./common/bazel.h" #include "./common/blob_file.h" #include "./common/defs.h" #include "./common/hash.h" @@ -271,92 +273,9 @@ int Fuzz(const Environment &env, const BinaryInfo &binary_info, return ExitCode(); } -struct TestShard { - int index = 0; - int total_shards = 1; -}; - -// https://bazel.build/reference/test-encyclopedia#initial-conditions -absl::Duration GetBazelTestTimeout() { - const char *test_timeout_env = std::getenv("TEST_TIMEOUT"); - if (test_timeout_env == nullptr) return absl::InfiniteDuration(); - int timeout_s = 0; - CHECK(absl::SimpleAtoi(test_timeout_env, &timeout_s)) - << "Failed to parse TEST_TIMEOUT: \"" << test_timeout_env << "\""; - return absl::Seconds(timeout_s); -} - -void ReportErrorWhenNotEnoughTimeToRunEverything(absl::Time start_time, - absl::Duration test_time_limit, - int executed_tests_in_shard, - int fuzz_test_count, - int shard_count) { - static const absl::Duration bazel_test_timeout = GetBazelTestTimeout(); - constexpr float kTimeoutSafetyFactor = 1.2; - const auto required_test_time = kTimeoutSafetyFactor * test_time_limit; - const auto remaining_duration = - bazel_test_timeout - (absl::Now() - start_time); - if (required_test_time <= remaining_duration) return; - std::string error = - "Cannot fuzz a fuzz test within the given timeout. Please "; - if (executed_tests_in_shard == 0) { - // Increasing number of shards won't help. - const absl::Duration suggested_timeout = - required_test_time * ((fuzz_test_count - 1) / shard_count + 1); - absl::StrAppend(&error, "set the `timeout` to ", suggested_timeout, - " or reduce the fuzzing time, "); - } else { - constexpr int kMaxShardCount = 50; - const int suggested_shard_count = std::min( - (fuzz_test_count - 1) / executed_tests_in_shard + 1, kMaxShardCount); - const int suggested_tests_per_shard = - (fuzz_test_count - 1) / suggested_shard_count + 1; - if (suggested_tests_per_shard > executed_tests_in_shard) { - // We wouldn't be able to execute the suggested number of tests without - // timeout. This case can only happen if we would in fact need more than - // `kMaxShardCount` shards, indicating that there are simply too many fuzz - // tests in a binary. - CHECK_EQ(suggested_shard_count, kMaxShardCount); - absl::StrAppend(&error, - "split the fuzz tests into several test binaries where " - "each binary has at most ", - executed_tests_in_shard * kMaxShardCount, "tests ", - "with `shard_count` = ", kMaxShardCount, ", "); - } else { - // In this case, `suggested_shard_count` must be greater than - // `shard_count`, otherwise we would have already executed all the tests - // without a timeout. - CHECK_GT(suggested_shard_count, shard_count); - absl::StrAppend(&error, "increase the `shard_count` to ", - suggested_shard_count, ", "); - } - } - absl::StrAppend(&error, "to avoid this issue. "); - absl::StrAppend(&error, - "(https://bazel.build/reference/be/" - "common-definitions#common-attributes-tests)"); - CHECK(false) << error; -} TestShard SetUpTestSharding() { - TestShard test_shard; - if (const char *test_total_shards_env = std::getenv("TEST_TOTAL_SHARDS"); - test_total_shards_env != nullptr) { - CHECK(absl::SimpleAtoi(test_total_shards_env, &test_shard.total_shards)) - << "Failed to parse TEST_TOTAL_SHARDS as an integer: \"" - << test_total_shards_env << "\""; - CHECK_GT(test_shard.total_shards, 0) - << "TEST_TOTAL_SHARDS must be greater than 0."; - } - if (const char *test_shard_index_env = std::getenv("TEST_SHARD_INDEX"); - test_shard_index_env != nullptr) { - CHECK(absl::SimpleAtoi(test_shard_index_env, &test_shard.index)) - << "Failed to parse TEST_SHARD_INDEX as an integer: \"" - << test_shard_index_env << "\""; - CHECK(0 <= test_shard.index && test_shard.index < test_shard.total_shards) - << "TEST_SHARD_INDEX must be in the range [0, " - << test_shard.total_shards << ")."; - } + TestShard test_shard = GetBazelTestShard(); // Update the shard status file to indicate that we support test sharding. // It suffices to update the file's modification time, but we clear the // contents for simplicity. This is also what the GoogleTest framework does. @@ -365,7 +284,6 @@ TestShard SetUpTestSharding() { test_shard_status_file != nullptr) { ClearLocalFileContents(test_shard_status_file); } - return test_shard; } @@ -432,22 +350,34 @@ void DeduplicateAndStoreNewCrashes( } } -// Seeds the corpus files in `env.workdir` with the previously distilled corpus -// files from `src_dir`. +// Seeds the corpus files in `env.workdir` with the inputs in `regression_dir` +// (always used) and the previously distilled corpus files from `coverage_dir` +// (used if non-empty). SeedCorpusConfig GetSeedCorpusConfig(const Environment &env, - std::string_view src_dir) { + std::string_view regression_dir, + std::string_view coverage_dir) { const WorkDir workdir{env}; + std::vector sources = {{ + .dir_glob = std::string(regression_dir), + .num_recent_dirs = 1, + .individual_input_rel_glob = "*", + .sampled_fraction_or_count = 1.0f, + }}; + if (!coverage_dir.empty()) { + sources.push_back(SeedCorpusSource{ + .dir_glob = std::string(coverage_dir), + .num_recent_dirs = 1, + // We're using the previously distilled corpus files as seeds. + .shard_rel_glob = + std::filesystem::path{ + workdir.DistilledCorpusFilePaths().AllShardsGlob()} + .filename(), + .individual_input_rel_glob = "*", + .sampled_fraction_or_count = 1.0f, + }); + } return { - .sources = {SeedCorpusSource{ - .dir_glob = std::string(src_dir), - .num_recent_dirs = 1, - // We're using the previously distilled corpus files as seeds. - .shard_rel_glob = - std::filesystem::path{ - workdir.DistilledCorpusFilePaths().AllShardsGlob()} - .filename(), - .sampled_fraction_or_count = 1.0f, - }}, + .sources = std::move(sources), .destination = { .dir_path = env.workdir, @@ -496,9 +426,7 @@ int UpdateCorpusDatabaseForFuzzTests( absl::Time start_time = absl::Now(); LOG(INFO) << "Starting the update of the corpus database for fuzz tests:" << "\nBinary: " << env.binary - << "\nCorpus database: " << fuzztest_config.corpus_database - << "\nFuzz tests: " - << absl::StrJoin(fuzztest_config.fuzz_tests, ", "); + << "\nCorpus database: " << fuzztest_config.corpus_database; // Step 1: Preliminary set up of test sharding, binary info, etc. const auto [test_shard_index, total_test_shards] = SetUpTestSharding(); @@ -515,14 +443,33 @@ int UpdateCorpusDatabaseForFuzzTests( absl::FormatTime("%Y-%m-%d-%H-%M-%S", absl::Now(), absl::UTCTimeZone()); return stamp; }(); - // The full workdir paths will be formed by appending the fuzz test names to - // the base workdir path. We use different path when only replaying to avoid - // replaying an unfinished fuzzing sessions. + std::vector fuzz_tests_to_run; + if (env.fuzztest_single_test_mode) { + CHECK(fuzztest_config.fuzz_tests_in_current_shard.size() == 1) + << "Must select exactly one fuzz test when running in the single test " + "mode"; + fuzz_tests_to_run = fuzztest_config.fuzz_tests_in_current_shard; + } else { + for (int i = 0; i < fuzztest_config.fuzz_tests.size(); ++i) { + if (i % total_test_shards == test_shard_index) { + fuzz_tests_to_run.push_back(fuzztest_config.fuzz_tests[i]); + } + } + } + LOG(INFO) << "Fuzz tests to run:" << absl::StrJoin(fuzz_tests_to_run, ", "); + + const bool is_workdir_specified = !env.workdir.empty(); + CHECK(!is_workdir_specified || env.fuzztest_single_test_mode); + // When env.workdir is empty, the full workdir paths will be formed by + // appending the fuzz test names to the base workdir path. We use different + // path when only replaying to avoid replaying an unfinished fuzzing sessions. const auto base_workdir_path = - corpus_database_path / - absl::StrFormat("workdir%s.%03d", - fuzztest_config.only_replay_corpus ? "-replay" : "", - test_shard_index); + is_workdir_specified + ? std::filesystem::path{} // Will not be used. + : corpus_database_path / + absl::StrFormat("workdir%s.%03d", + fuzztest_config.only_replay ? "-replay" : "", + test_shard_index); // There's no point in saving the binary info to the workdir, since the // workdir is deleted at the end. env.save_binary_info = false; @@ -537,35 +484,74 @@ int UpdateCorpusDatabaseForFuzzTests( // Find the last index of a fuzz test for which we already have a workdir. bool is_resuming = false; int resuming_fuzztest_idx = 0; - for (int i = 0; i < fuzztest_config.fuzz_tests.size(); ++i) { - if (i % total_test_shards != test_shard_index) continue; - env.workdir = base_workdir_path / fuzztest_config.fuzz_tests[i]; - // Check the existence of the coverage path to not only make sure the - // workdir exists, but also that it was created for the same binary as in - // this run. - if (RemotePathExists(WorkDir{env}.CoverageDirPath())) { - is_resuming = true; - resuming_fuzztest_idx = i; + if (!fuzztest_config.execution_id.has_value()) { + for (int i = 0; i < fuzz_tests_to_run.size(); ++i) { + if (!is_workdir_specified) { + env.workdir = base_workdir_path / fuzz_tests_to_run[i]; + } + // Check the existence of the coverage path to not only make sure the + // workdir exists, but also that it was created for the same binary as in + // this run. + if (RemotePathExists(WorkDir{env}.CoverageDirPath())) { + is_resuming = true; + resuming_fuzztest_idx = i; + } } } LOG_IF(INFO, is_resuming) << "Resuming from the fuzz test " - << fuzztest_config.fuzz_tests[resuming_fuzztest_idx] + << fuzz_tests_to_run[resuming_fuzztest_idx] << " (index: " << resuming_fuzztest_idx << ")"; // Step 3: Iterate over the fuzz tests and run them. const std::string binary = env.binary; - for (int i = resuming_fuzztest_idx; i < fuzztest_config.fuzz_tests.size(); - ++i) { - if (i % total_test_shards != test_shard_index) continue; - if (fuzztest_config.GetTimeLimitPerTest() < absl::InfiniteDuration()) { - // TODO(fniksic): Test this behavior in end-to-end tests. - ReportErrorWhenNotEnoughTimeToRunEverything( - start_time, fuzztest_config.GetTimeLimitPerTest(), - /*executed_tests_in_shard=*/i / total_test_shards, - fuzztest_config.fuzz_tests.size(), total_test_shards); + for (int i = resuming_fuzztest_idx; i < fuzz_tests_to_run.size(); ++i) { + if (!env.fuzztest_single_test_mode && + fuzztest_config.GetTimeLimitPerTest() < absl::InfiniteDuration()) { + const absl::Duration test_time_limit = + fuzztest_config.GetTimeLimitPerTest(); + const absl::Status has_enough_time = CheckBazelHasEnoughTimeToRunTest( + start_time, test_time_limit, + /*executed_tests_in_shard=*/i, fuzztest_config.fuzz_tests.size()); + if (!has_enough_time.ok()) { + LOG(ERROR) << "Not enough time for running the fuzz test " + << fuzz_tests_to_run[i] << " for " << test_time_limit + << ". Still running the test anyway. Error: " + << absl::StrCat(has_enough_time); + } + } + if (!is_workdir_specified) { + env.workdir = base_workdir_path / fuzz_tests_to_run[i]; + CHECK_OK(RemoteMkdir(env.workdir)); + } + const auto execution_id_path = + (base_workdir_path / + absl::StrCat(fuzz_tests_to_run[i], ".execution_id")) + .string(); + if (!is_workdir_specified && fuzztest_config.execution_id.has_value()) { + is_resuming = RemotePathExists(WorkDir{env}.CoverageDirPath()); + const bool execution_id_matched = [&] { + if (!RemotePathExists(execution_id_path)) return false; + CHECK(!RemotePathIsDirectory(execution_id_path)); + std::string prev_execution_id; + CHECK_OK(RemoteFileGetContents(execution_id_path, prev_execution_id)); + LOG(INFO) << "Got previous execution id " << prev_execution_id; + return prev_execution_id == *fuzztest_config.execution_id; + }(); + if (execution_id_matched) { + LOG(INFO) << "execution id matches for " << fuzz_tests_to_run[i]; + // If execution ID matches but the previous coverage is missing, it + // means the test was previously finished. + if (!is_resuming) { + LOG(INFO) << "Skipping running the fuzz test " + << fuzz_tests_to_run[i]; + continue; + } + LOG(INFO) << "Resuming running the fuzz test " << fuzz_tests_to_run[i]; + } else { + is_resuming = false; + } } - env.workdir = base_workdir_path / fuzztest_config.fuzz_tests[i]; if (RemotePathExists(env.workdir) && !is_resuming) { // This could be a workdir from a failed run that used a different version // of the binary. We delete it so that we don't have to deal with the @@ -573,29 +559,52 @@ int UpdateCorpusDatabaseForFuzzTests( CHECK_OK(RemotePathDelete(env.workdir, /*recursively=*/true)); } const WorkDir workdir{env}; - CHECK_OK(RemoteMkdir( - workdir.CoverageDirPath())); // Implicitly creates the workdir + CHECK_OK(RemoteMkdir(workdir.CoverageDirPath())); + + // Updating execution ID must be after creating the coverage dir. Otherwise + // if it fails to create coverage dir after updating execution ID, next + // attempt would skip this test. + if (!is_workdir_specified && fuzztest_config.execution_id.has_value() && + !is_resuming) { + CHECK_OK(RemoteFileSetContents(execution_id_path, + *fuzztest_config.execution_id)); + } + + absl::Cleanup clean_up_workdir = [is_workdir_specified, &env] { + if (!is_workdir_specified) { + CHECK_OK(RemotePathDelete(env.workdir, /*recursively=*/true)); + } + }; - // Seed the fuzzing session with the latest coverage corpus from the - // previous fuzzing session. const std::filesystem::path fuzztest_db_path = - corpus_database_path / fuzztest_config.fuzz_tests[i]; + corpus_database_path / fuzz_tests_to_run[i]; + const std::filesystem::path regression_dir = + fuzztest_db_path / "regression"; const std::filesystem::path coverage_dir = fuzztest_db_path / "coverage"; - if (RemotePathExists(coverage_dir.c_str()) && !is_resuming) { + + // Seed the fuzzing session with the latest coverage corpus and regression + // inputs from the previous fuzzing session. + if (!is_resuming) { CHECK_OK(GenerateSeedCorpusFromConfig( - GetSeedCorpusConfig(env, coverage_dir.c_str()), env.binary_name, - env.binary_hash)); + GetSeedCorpusConfig(env, regression_dir.c_str(), + fuzztest_config.replay_coverage_inputs + ? coverage_dir.c_str() + : ""), + env.binary_name, env.binary_hash)) + << "while generating the seed corpus"; } - // TODO: b/338217594 - Call the FuzzTest binary in a flag-agnostic way. - constexpr std::string_view kFuzzTestFuzzFlag = "--fuzz="; - constexpr std::string_view kFuzzTestReplayCorpusFlag = - "--replay_corpus="; - std::string_view test_selection_flag = fuzztest_config.only_replay_corpus - ? kFuzzTestReplayCorpusFlag - : kFuzzTestFuzzFlag; - env.binary = absl::StrCat(binary, " ", test_selection_flag, - fuzztest_config.fuzz_tests[i]); + if (!env.fuzztest_single_test_mode) { + // TODO: b/338217594 - Call the FuzzTest binary in a flag-agnostic way. + constexpr std::string_view kFuzzTestFuzzFlag = "--fuzz="; + constexpr std::string_view kFuzzTestReplayCorpusFlag = + "--replay_corpus="; + std::string_view test_selection_flag = fuzztest_config.only_replay + ? kFuzzTestReplayCorpusFlag + : kFuzzTestFuzzFlag; + env.binary = + absl::StrCat(binary, " ", test_selection_flag, fuzz_tests_to_run[i]); + } absl::Duration time_limit = fuzztest_config.GetTimeLimitPerTest(); absl::Duration time_spent = absl::ZeroDuration(); @@ -607,9 +616,8 @@ int UpdateCorpusDatabaseForFuzzTests( } is_resuming = false; - LOG(INFO) << (fuzztest_config.only_replay_corpus ? "Replaying " - : "Fuzzing ") - << fuzztest_config.fuzz_tests[i] << " for " << time_limit + LOG(INFO) << (fuzztest_config.only_replay ? "Replaying " : "Fuzzing ") + << fuzz_tests_to_run[i] << " for " << time_limit << "\n\tTest binary: " << env.binary; const absl::Time start_time = absl::Now(); @@ -621,7 +629,7 @@ int UpdateCorpusDatabaseForFuzzTests( record_fuzzing_time.Stop(); if (!stats_root_path.empty()) { - const auto stats_dir = stats_root_path / fuzztest_config.fuzz_tests[i]; + const auto stats_dir = stats_root_path / fuzz_tests_to_run[i]; CHECK_OK(RemoteMkdir(stats_dir.c_str())); CHECK_OK(RemotePathRename( workdir.FuzzingStatsPath(), @@ -629,7 +637,9 @@ int UpdateCorpusDatabaseForFuzzTests( .c_str())); } - if (fuzztest_config.only_replay_corpus) continue; + // TODO(xinhaoyuan): Have a separate flag to skip corpus updating instead + // of checking whether workdir is specified or not. + if (fuzztest_config.only_replay || is_workdir_specified) continue; // Distill and store the coverage corpus. Distill(env); @@ -657,10 +667,6 @@ int UpdateCorpusDatabaseForFuzzTests( DeduplicateAndStoreNewCrashes(crashing_dir, workdir, env.total_shards, std::move(crash_metadata)); } - // Path may not exist if there are no fuzz tests in the shard. - if (RemotePathExists(base_workdir_path.c_str())) { - CHECK_OK(RemotePathDelete(base_workdir_path.c_str(), /*recursively=*/true)); - } return EXIT_SUCCESS; } @@ -724,7 +730,7 @@ int CentipedeMain(const Environment &env, << "Failed to deserialize target configuration"; if (!target_config->corpus_database.empty()) { const auto time_limit_per_test = target_config->GetTimeLimitPerTest(); - CHECK(target_config->only_replay_corpus || + CHECK(target_config->only_replay || time_limit_per_test < absl::InfiniteDuration()) << "Updating corpus database requires specifying time limit per " "fuzz test."; diff --git a/centipede/environment.cc b/centipede/environment.cc index 10bd8ee7..3b2a7f38 100644 --- a/centipede/environment.cc +++ b/centipede/environment.cc @@ -261,7 +261,8 @@ void Environment::UpdateWithTargetConfig( // Update `timeout_per_input` and consequently `timeout_per_batch`. const size_t time_limit_per_input_sec = convert_to_seconds(config.time_limit_per_input, "Time limit per input"); - CHECK(timeout_per_input == Default().timeout_per_input || + CHECK(timeout_per_input == 0 || + timeout_per_input == Default().timeout_per_input || timeout_per_input == time_limit_per_input_sec) << "Value for --timeout_per_input is inconsistent with the value for " "time_limit_per_input in the target binary:" @@ -304,7 +305,7 @@ void Environment::UpdateWithTargetConfig( << VV(stack_limit_kb) << VV(config.stack_limit); stack_limit_kb = bytes_to_kb(config.stack_limit); - if (config.only_replay_corpus) { + if (config.only_replay) { load_shards_only = true; populate_binary_info = false; } diff --git a/centipede/environment.h b/centipede/environment.h index 751fad9e..79a71b10 100644 --- a/centipede/environment.h +++ b/centipede/environment.h @@ -127,6 +127,9 @@ struct Environment { bool first_corpus_dir_output_only = false; // If set, load/merge shards without fuzzing new inputs. bool load_shards_only = false; + // If set, operate on the corpus database for a single test specified by + // FuzzTest instead of all the tests. + bool fuzztest_single_test_mode = false; // Command line-related fields ----------------------------------------------- diff --git a/centipede/environment_test.cc b/centipede/environment_test.cc index 2f788a68..68e5bdca 100644 --- a/centipede/environment_test.cc +++ b/centipede/environment_test.cc @@ -198,7 +198,7 @@ TEST(Environment, DiesOnInconsistentStackLimitKbAndTargetConfigStackLimit) { TEST(Environment, UpdatesReplayOnlyConfiguration) { Environment env; - fuzztest::internal::Configuration config{.only_replay_corpus = true}; + fuzztest::internal::Configuration config{.only_replay = true}; env.UpdateWithTargetConfig(config); EXPECT_TRUE(env.load_shards_only); EXPECT_FALSE(env.populate_binary_info); diff --git a/common/BUILD b/common/BUILD index 8090a6e1..4f156347 100644 --- a/common/BUILD +++ b/common/BUILD @@ -35,6 +35,18 @@ exports_files( ### Libraries +cc_library( + name = "bazel", + srcs = ["bazel.cc"], + hdrs = ["bazel.h"], + deps = [ + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/time", + ], +) + cc_library( name = "blob_file", srcs = ["blob_file.cc"], diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 8279faee..470c42e7 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -19,6 +19,20 @@ endif() ### Libraries +fuzztest_cc_library( + NAME + bazel + HDRS + "bazel.h" + SRCS + "bazel.cc" + DEPS + absl::check + absl::strings + absl::status + absl::time +) + fuzztest_cc_library( NAME blob_file diff --git a/common/bazel.cc b/common/bazel.cc new file mode 100644 index 00000000..ee002f42 --- /dev/null +++ b/common/bazel.cc @@ -0,0 +1,120 @@ +// Copyright 2024 The Centipede Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "./common/bazel.h" + +#include +#include +#include + +#include "absl/log/check.h" +#include "absl/status/status.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" + +namespace centipede { + +namespace { + +absl::Duration GetBazelTestTimeout() { + const char *test_timeout_env = std::getenv("TEST_TIMEOUT"); + if (test_timeout_env == nullptr) return absl::InfiniteDuration(); + int timeout_s = 0; + CHECK(absl::SimpleAtoi(test_timeout_env, &timeout_s)) + << "Failed to parse TEST_TIMEOUT: \"" << test_timeout_env << "\""; + return absl::Seconds(timeout_s); +} + +} // namespace + +const TestShard &GetBazelTestShard() { + static TestShard cached_test_shard = [] { + TestShard test_shard; + if (const char *test_total_shards_env = std::getenv("TEST_TOTAL_SHARDS"); + test_total_shards_env != nullptr) { + CHECK(absl::SimpleAtoi(test_total_shards_env, &test_shard.total_shards)) + << "Failed to parse TEST_TOTAL_SHARDS as an integer: \"" + << test_total_shards_env << "\""; + CHECK_GT(test_shard.total_shards, 0) + << "TEST_TOTAL_SHARDS must be greater than 0."; + } + if (const char *test_shard_index_env = std::getenv("TEST_SHARD_INDEX"); + test_shard_index_env != nullptr) { + CHECK(absl::SimpleAtoi(test_shard_index_env, &test_shard.index)) + << "Failed to parse TEST_SHARD_INDEX as an integer: \"" + << test_shard_index_env << "\""; + CHECK(0 <= test_shard.index && test_shard.index < test_shard.total_shards) + << "TEST_SHARD_INDEX must be in the range [0, " + << test_shard.total_shards << ")."; + } + return test_shard; + }(); + return cached_test_shard; +} + +absl::Status CheckBazelHasEnoughTimeToRunTest(absl::Time target_start_time, + absl::Duration test_time_limit, + int executed_tests_in_shard, + int fuzz_test_count) { + static const absl::Duration bazel_test_timeout = GetBazelTestTimeout(); + const int shard_count = GetBazelTestShard().total_shards; + constexpr float kTimeoutSafetyFactor = 1.2; + const auto required_test_time = kTimeoutSafetyFactor * test_time_limit; + const auto remaining_duration = + bazel_test_timeout - (absl::Now() - target_start_time); + if (required_test_time <= remaining_duration) return absl::OkStatus(); + std::string error = + "Cannot fuzz a fuzz test within the given timeout. Please "; + if (executed_tests_in_shard == 0) { + // Increasing number of shards won't help. + const absl::Duration suggested_timeout = + required_test_time * ((fuzz_test_count - 1) / shard_count + 1); + absl::StrAppend(&error, "set the `timeout` to ", suggested_timeout, + " or reduce the fuzzing time, "); + } else { + constexpr int kMaxShardCount = 50; + const int suggested_shard_count = std::min( + (fuzz_test_count - 1) / executed_tests_in_shard + 1, kMaxShardCount); + const int suggested_tests_per_shard = + (fuzz_test_count - 1) / suggested_shard_count + 1; + if (suggested_tests_per_shard > executed_tests_in_shard) { + // We wouldn't be able to execute the suggested number of tests without + // timeout. This case can only happen if we would in fact need more than + // `kMaxShardCount` shards, indicating that there are simply too many fuzz + // tests in a binary. + CHECK_EQ(suggested_shard_count, kMaxShardCount); + absl::StrAppend(&error, + "split the fuzz tests into several test binaries where " + "each binary has at most ", + executed_tests_in_shard * kMaxShardCount, "tests ", + "with `shard_count` = ", kMaxShardCount, ", "); + } else { + // In this case, `suggested_shard_count` must be greater than + // `shard_count`, otherwise we would have already executed all the tests + // without a timeout. + CHECK_GT(suggested_shard_count, shard_count); + absl::StrAppend(&error, "increase the `shard_count` to ", + suggested_shard_count, ", "); + } + } + absl::StrAppend(&error, "to avoid this issue. "); + absl::StrAppend(&error, + "(https://bazel.build/reference/be/" + "common-definitions#common-attributes-tests)"); + return absl::ResourceExhaustedError(error); +} + +} // namespace centipede diff --git a/common/bazel.h b/common/bazel.h new file mode 100644 index 00000000..cbd06824 --- /dev/null +++ b/common/bazel.h @@ -0,0 +1,44 @@ +// Copyright 2024 The Centipede Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef FUZZTEST_COMMON_BAZEL_H_ +#define FUZZTEST_COMMON_BAZEL_H_ + +#include "absl/status/status.h" +#include "absl/time/time.h" + +namespace centipede { + +struct TestShard { + int index = 0; + int total_shards = 1; +}; + +// Get Bazel sharding information based on // +// https://bazel.build/reference/test-encyclopedia#initial-conditions +const TestShard& GetBazelTestShard(); + +// Checks and returns Ok if there is enough time left to run the current test +// for `test_time_limit` given `target_start_time` and timeout and sharding +// information from Bazel, or returns an error status otherwise. It uses +// `executed_tests_in_shard` and `fuzz_test_count` to include suggestions into +// the error status message. +absl::Status CheckBazelHasEnoughTimeToRunTest(absl::Time target_start_time, + absl::Duration test_time_limit, + int executed_tests_in_shard, + int fuzz_test_count); + +} // namespace centipede + +#endif // FUZZTEST_COMMON_BAZEL_H_ diff --git a/e2e_tests/BUILD b/e2e_tests/BUILD index f16b1bf9..d7cec635 100644 --- a/e2e_tests/BUILD +++ b/e2e_tests/BUILD @@ -100,6 +100,7 @@ cc_binary( # Must be run with `--config=fuzztest-experimental --config=asan`. cc_test( name = "corpus_database_test", + size = "large", srcs = ["corpus_database_test.cc"], data = [ "@com_google_fuzztest//centipede:centipede_uninstrumented", @@ -125,6 +126,7 @@ cc_test( "@com_google_absl//absl/time", "@com_google_fuzztest//centipede:weak_sancov_stubs", "@com_google_fuzztest//fuzztest:io", + "@com_google_fuzztest//fuzztest:logging", "@com_google_fuzztest//fuzztest:subprocess", "@com_google_googletest//:gtest_main", ], diff --git a/e2e_tests/corpus_database_test.cc b/e2e_tests/corpus_database_test.cc index d2d356e5..ade31083 100644 --- a/e2e_tests/corpus_database_test.cc +++ b/e2e_tests/corpus_database_test.cc @@ -16,6 +16,7 @@ #include // NOLINT #include #include +#include #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -29,6 +30,7 @@ #include "absl/time/time.h" #include "./e2e_tests/test_binary_util.h" #include "./fuzztest/internal/io.h" +#include "./fuzztest/internal/logging.h" #include "./fuzztest/internal/subprocess.h" namespace fuzztest::internal { @@ -52,7 +54,13 @@ absl::StatusOr FindFile(absl::string_view root_path, return absl::NotFoundError(absl::StrCat("File ", file_name, " not found.")); } -class UpdateCorpusDatabaseTest : public testing::Test { +enum class ExecutionModelParam { + kSingleBinary, + kWithCentipedeBinary, +}; + +class UpdateCorpusDatabaseTest + : public ::testing::TestWithParam { protected: static void SetUpTestSuite() { #if defined(__has_feature) @@ -65,21 +73,20 @@ class UpdateCorpusDatabaseTest : public testing::Test { "Please run with --config=fuzztest-experimental."; #endif #endif + CHECK(temp_dir_ == nullptr); + } + static void RunUpdateCorpusDatabase() { + if (temp_dir_ != nullptr) return; temp_dir_ = new TempDir(); - - auto [status, std_out, std_err] = RunBinary( - CentipedePath(), - {.flags = {{"binary", - absl::StrJoin({GetCorpusDatabaseTestingBinaryPath(), - CreateFuzzTestFlag("corpus_database", - GetCorpusDatabasePath()), - CreateFuzzTestFlag("fuzz_for", "30s"), - CreateFuzzTestFlag("jobs", "2")}, - /*separator=*/" ")}}}); - - *centipede_std_out_ = std::move(std_out); - *centipede_std_err_ = std::move(std_err); + auto [status, std_out, std_err] = RunBinaryMaybeWithCentipede( + GetCorpusDatabaseTestingBinaryPath(), + {.fuzztest_flags = { + {"corpus_database", GetCorpusDatabasePath()}, + {"fuzz_for", "30s"}, + {"jobs", "2"}, + }}); + *update_corpus_database_std_err_ = std::move(std_err); } static void TearDownTestSuite() { @@ -88,81 +95,101 @@ class UpdateCorpusDatabaseTest : public testing::Test { } static std::string GetCorpusDatabasePath() { - CHECK(temp_dir_ != nullptr); + RunUpdateCorpusDatabase(); return std::filesystem::path(temp_dir_->dirname()) / "corpus_database"; } - static absl::string_view GetCentipedeStdOut() { return *centipede_std_out_; } + static absl::string_view GetUpdateCorpusDatabaseStdErr() { + RunUpdateCorpusDatabase(); + return *update_corpus_database_std_err_; + } - static absl::string_view GetCentipedeStdErr() { return *centipede_std_err_; } + static RunResults RunBinaryMaybeWithCentipede(absl::string_view binary_path, + const RunOptions &options) { + switch (GetParam()) { + case ExecutionModelParam::kSingleBinary: + return RunBinary(binary_path, options); + case ExecutionModelParam::kWithCentipedeBinary: { + RunOptions centipede_options; + centipede_options.env = options.env; + centipede_options.timeout = options.timeout; + std::vector binary_args; + binary_args.push_back(std::string(binary_path)); + for (const auto &[key, value] : options.fuzztest_flags) { + binary_args.push_back(CreateFuzzTestFlag(key, value)); + } + for (const auto &[key, value] : options.flags) { + binary_args.push_back(absl::StrCat("--", key, "=", value)); + } + centipede_options.flags = { + {"binary", absl::StrJoin(binary_args, " ")}, + // Disable symbolization to more quickly get to fuzzing. + {"symbolizer_path", ""}, + }; + return RunBinary(CentipedePath(), centipede_options); + } + } + FUZZTEST_INTERNAL_CHECK(false, "Unsupported execution model!\n"); + } private: static TempDir *temp_dir_; - static absl::NoDestructor centipede_std_out_; - static absl::NoDestructor centipede_std_err_; + static absl::NoDestructor update_corpus_database_std_err_; }; TempDir *UpdateCorpusDatabaseTest::temp_dir_ = nullptr; -absl::NoDestructor UpdateCorpusDatabaseTest::centipede_std_out_{}; -absl::NoDestructor UpdateCorpusDatabaseTest::centipede_std_err_{}; +absl::NoDestructor + UpdateCorpusDatabaseTest::update_corpus_database_std_err_{}; -TEST_F(UpdateCorpusDatabaseTest, RunsFuzzTests) { - EXPECT_THAT(GetCentipedeStdErr(), +TEST_P(UpdateCorpusDatabaseTest, RunsFuzzTests) { + EXPECT_THAT(GetUpdateCorpusDatabaseStdErr(), AllOf(HasSubstr("Fuzzing FuzzTest.FailsInTwoWays"), HasSubstr("Fuzzing FuzzTest.FailsWithStackOverflow"))); } -TEST_F(UpdateCorpusDatabaseTest, UsesMultipleShardsForFuzzingAndDistillation) { +TEST_P(UpdateCorpusDatabaseTest, UsesMultipleShardsForFuzzingAndDistillation) { EXPECT_THAT( - GetCentipedeStdErr(), + GetUpdateCorpusDatabaseStdErr(), AllOf(HasSubstr("[S0.0] begin-fuzz"), HasSubstr("[S1.0] begin-fuzz"), HasSubstr("DISTILL[S.0]: Distilling to output shard 0"), HasSubstr("DISTILL[S.1]: Distilling to output shard 1"))); } -TEST_F(UpdateCorpusDatabaseTest, FindsAllCrashes) { +TEST_P(UpdateCorpusDatabaseTest, FindsAllCrashes) { EXPECT_THAT( - GetCentipedeStdErr(), + GetUpdateCorpusDatabaseStdErr(), AllOf(ContainsRegex(R"re(Failure\s*: GoogleTest assertion failure)re"), ContainsRegex(R"re(Failure\s*: heap-buffer-overflow)re"), ContainsRegex(R"re(Failure\s*: stack-limit-exceeded)re"))); } -TEST_F(UpdateCorpusDatabaseTest, ResumedFuzzTestRunsForRemainingTime) { +TEST_P(UpdateCorpusDatabaseTest, ResumedFuzzTestRunsForRemainingTime) { TempDir corpus_database; // 1st run that gets interrupted. - auto [fst_status, fst_std_out, fst_std_err] = RunBinary( - CentipedePath(), - {.flags = {{"binary", - absl::StrJoin({GetCorpusDatabaseTestingBinaryPath(), - CreateFuzzTestFlag("corpus_database", - corpus_database.dirname()), - CreateFuzzTestFlag("fuzz_for", "300s")}, - /*separator=*/" ")}, - // Disable symbolization to more quickly get to fuzzing. - {"symbolizer_path", ""}}, - // Stop the binary with SIGTERM before the fuzzing is done. + auto [fst_status, fst_std_out, fst_std_err] = RunBinaryMaybeWithCentipede( + GetCorpusDatabaseTestingBinaryPath(), + {.fuzztest_flags = + { + {"corpus_database", corpus_database.dirname()}, + {"fuzz_for", "300s"}, + }, .timeout = absl::Seconds(10)}); - ASSERT_THAT(fst_status, Eq(Signal(SIGTERM))); // Adjust the fuzzing time so that only 1s remains. const absl::StatusOr fuzzing_time_file = FindFile(corpus_database.dirname(), "fuzzing_time"); - ASSERT_TRUE(fuzzing_time_file.ok()); + ASSERT_TRUE(fuzzing_time_file.ok()) << fst_std_err; ASSERT_TRUE(WriteFile(*fuzzing_time_file, "299s")); // 2nd run that resumes the fuzzing. - auto [snd_status, snd_std_out, snd_std_err] = RunBinary( - CentipedePath(), - {.flags = {{"binary", - absl::StrJoin({GetCorpusDatabaseTestingBinaryPath(), - CreateFuzzTestFlag("corpus_database", - corpus_database.dirname()), - CreateFuzzTestFlag("fuzz_for", "300s")}, - /*separator=*/" ")}, - // Disable symbolization to more quickly get to fuzzing. - {"symbolizer_path", ""}}, + auto [snd_status, snd_std_out, snd_std_err] = RunBinaryMaybeWithCentipede( + GetCorpusDatabaseTestingBinaryPath(), + {.fuzztest_flags = + { + {"corpus_database", corpus_database.dirname()}, + {"fuzz_for", "300s"}, + }, .timeout = absl::Seconds(10)}); EXPECT_THAT( @@ -172,16 +199,94 @@ TEST_F(UpdateCorpusDatabaseTest, ResumedFuzzTestRunsForRemainingTime) { HasSubstr("Fuzzing FuzzTest.FailsInTwoWays for 1s"))); } -TEST_F(UpdateCorpusDatabaseTest, ReplaysFuzzTestsInParallel) { - auto [status, std_out, std_err] = RunBinary( - CentipedePath(), - {.flags = {{"binary", - absl::StrJoin({GetCorpusDatabaseTestingBinaryPath(), - CreateFuzzTestFlag("corpus_database", - GetCorpusDatabasePath()), - CreateFuzzTestFlag("replay_corpus_for", "inf"), - CreateFuzzTestFlag("jobs", "2")}, - /*separator=*/" ")}}, +TEST_P(UpdateCorpusDatabaseTest, + ResumesOrSkipsFuzzTestRunsWhenExecutionIdsMatch) { + TempDir corpus_database; + + // 1st run that gets interrupted. + auto [fst_status, fst_std_out, fst_std_err] = RunBinaryMaybeWithCentipede( + GetCorpusDatabaseTestingBinaryPath(), + {.fuzztest_flags = + { + {"corpus_database", corpus_database.dirname()}, + {"fuzz_for", "300s"}, + {"execution_id", "1"}, + }, + .timeout = absl::Seconds(10)}); + + // 2nd run that should not resume due to the different execution ID + auto [snd_status, snd_std_out, snd_std_err] = RunBinaryMaybeWithCentipede( + GetCorpusDatabaseTestingBinaryPath(), + {.fuzztest_flags = + { + {"corpus_database", corpus_database.dirname()}, + {"fuzz_for", "300s"}, + {"execution_id", "2"}, + }, + .timeout = absl::Seconds(10)}); + EXPECT_THAT(snd_std_err, Not(HasSubstr("Resuming running the fuzz test"))) + << snd_std_err; + + // Adjust the fuzzing time so that only 1s remains. + const absl::StatusOr fuzzing_time_file = + FindFile(corpus_database.dirname(), "fuzzing_time"); + ASSERT_TRUE(fuzzing_time_file.ok()) << fst_std_err; + ASSERT_TRUE(WriteFile(*fuzzing_time_file, "299s")); + + // 3rd run that should resume due to the same execution ID + auto [thd_status, thd_std_out, thd_std_err] = RunBinaryMaybeWithCentipede( + GetCorpusDatabaseTestingBinaryPath(), + {.fuzztest_flags = + { + {"corpus_database", corpus_database.dirname()}, + {"fuzz_for", "300s"}, + {"execution_id", "2"}, + }, + .timeout = absl::Seconds(10)}); + EXPECT_THAT( + thd_std_err, + // The resumed fuzz test is the first one defined in the binary. + AllOf(HasSubstr("Resuming running the fuzz test FuzzTest.FailsInTwoWays"), + HasSubstr("Fuzzing FuzzTest.FailsInTwoWays for 1s"))) + << thd_std_err; + + // 4th run that should skip the test due the test is finished in the 3td + // exeuction with the same ID + auto [fth_status, fth_std_out, fth_std_err] = RunBinaryMaybeWithCentipede( + GetCorpusDatabaseTestingBinaryPath(), + {.fuzztest_flags = + { + {"corpus_database", corpus_database.dirname()}, + {"fuzz_for", "300s"}, + {"execution_id", "2"}, + }, + .timeout = absl::Seconds(10)}); + EXPECT_THAT( + fth_std_err, + // The skipped fuzz test is the first one defined in the binary. + HasSubstr("Skipping running the fuzz test FuzzTest.FailsInTwoWays")) + << fth_std_err; + + // 5th run that should not skip the test due the different execution ID + auto [fif_status, fif_std_out, fif_std_err] = RunBinaryMaybeWithCentipede( + GetCorpusDatabaseTestingBinaryPath(), + {.fuzztest_flags = + { + {"corpus_database", corpus_database.dirname()}, + {"fuzz_for", "300s"}, + {"execution_id", "3"}, + }, + .timeout = absl::Seconds(10)}); + EXPECT_THAT(fif_std_err, Not(HasSubstr("Skipping running the fuzz test"))) + << fif_std_err; +} + +TEST_P(UpdateCorpusDatabaseTest, ReplaysFuzzTestsInParallel) { + auto [status, std_out, std_err] = RunBinaryMaybeWithCentipede( + GetCorpusDatabaseTestingBinaryPath(), + {.fuzztest_flags = {{"corpus_database", GetCorpusDatabasePath()}, + {"replay_corpus_for", "inf"}, + {"jobs", "2"}}, .timeout = absl::Seconds(30)}); EXPECT_THAT( @@ -191,5 +296,25 @@ TEST_F(UpdateCorpusDatabaseTest, ReplaysFuzzTestsInParallel) { HasSubstr("[S0.0] begin-fuzz"), HasSubstr("[S1.0] begin-fuzz"))); } +TEST_P(UpdateCorpusDatabaseTest, PrintsErrorsWhenBazelTimeoutIsNotEnough) { + auto [status, std_out, std_err] = RunBinaryMaybeWithCentipede( + GetCorpusDatabaseTestingBinaryPath(), + { + .fuzztest_flags = {{"corpus_database", GetCorpusDatabasePath()}, + {"fuzz_for", "20s"}}, + .env = {{"TEST_TIMEOUT", "30"}}, + .timeout = absl::Seconds(40), + }); + EXPECT_THAT(std_err, AllOf(HasSubstr("Fuzzing FuzzTest.FailsInTwoWays"), + HasSubstr("Not enough time for running the fuzz " + "test FuzzTest.FailsWithStackOverflow"))) + << std_err; +} + +INSTANTIATE_TEST_SUITE_P( + UpdateCorpusDatabaseTestWithExecutionModel, UpdateCorpusDatabaseTest, + testing::ValuesIn({ExecutionModelParam::kSingleBinary, + ExecutionModelParam::kWithCentipedeBinary})); + } // namespace } // namespace fuzztest::internal diff --git a/e2e_tests/functional_test.cc b/e2e_tests/functional_test.cc index d5102528..ac619282 100644 --- a/e2e_tests/functional_test.cc +++ b/e2e_tests/functional_test.cc @@ -20,6 +20,7 @@ #include // NOLINT #include #include +#include #include #include #include @@ -69,12 +70,23 @@ constexpr absl::string_view kDefaultTargetBinary = absl::flat_hash_map WithTestSanitizerOptions( absl::flat_hash_map env) { if (!env.contains("ASAN_OPTIONS")) - env["ASAN_OPTIONS"] = "handle_abort=0:handle_sigfpe=0"; + env["ASAN_OPTIONS"] = "handle_abort=0:handle_sigfpe=0:detect_leaks=0"; if (!env.contains("MSAN_OPTIONS")) env["MSAN_OPTIONS"] = "handle_abort=0:handle_sigfpe=0"; return env; } +void ExpectTargetAbort(TerminationStatus status, absl::string_view std_err) { +#ifdef FUZZTEST_USE_CENTIPEDE + EXPECT_THAT(status, Ne(ExitCode(0))); + EXPECT_TRUE( + RE2::PartialMatch(std_err, absl::StrCat("Exit code\\s*:\\s*", SIGABRT))) + << std_err; +#else + EXPECT_THAT(status, Eq(Signal(SIGABRT))); +#endif +} + int CountSubstrs(absl::string_view haystack, absl::string_view needle) { int count = 0; while (true) { @@ -97,10 +109,12 @@ class UnitTestModeTest : public ::testing::Test { absl::string_view test_filter, absl::string_view target_binary = kDefaultTargetBinary, const absl::flat_hash_map& env = {}, - const absl::flat_hash_map& fuzzer_flags = {}) { + absl::flat_hash_map fuzzer_flags = {}) { + fuzzer_flags["print_subprocess_log"] = "true"; return RunBinary( BinaryPath(target_binary), - {.flags = {{GTEST_FLAG_PREFIX_ "filter", std::string(test_filter)}}, + {.flags = {{GTEST_FLAG_PREFIX_ "filter", std::string(test_filter)}, + {"symbolize_stacktrace", "0"}}, .fuzztest_flags = fuzzer_flags, .env = WithTestSanitizerOptions(env)}); } @@ -257,7 +271,7 @@ TEST_F(UnitTestModeTest, GoogleTestStaticTestSuiteFunctionsCalledInBalance) { TEST_F(UnitTestModeTest, GoogleTestWorksWithProtoExtensionsUsedInSeeds) { auto [status, std_out, std_err] = Run("MySuite.CheckProtoExtensions"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("Uses proto extensions")); } @@ -274,7 +288,7 @@ TEST_F(UnitTestModeTest, RepeatedFieldsHaveMinSizeWhenInitialized) { TEST_F(UnitTestModeTest, OptionalProtoFieldCanHaveNoValue) { auto [status, std_out, std_err] = Run("MySuite.FailsWhenFieldI32HasNoValue"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(UnitTestModeTest, OptionalProtoFieldThatIsUnsetNeverHasValue) { @@ -318,26 +332,26 @@ TEST_F(UnitTestModeTest, TEST_F(UnitTestModeTest, RequiredProtoFieldThatIsNotAlwaysSetCanHaveNoValue) { auto [status, std_out, std_err] = Run("MySuite.FailsWhenRequiredEnumFieldHasNoValue"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("cannot have null values")); } TEST_F(UnitTestModeTest, OptionalProtoFieldThatIsNotAlwaysSetCanHaveNoValue) { auto [status, std_out, std_err] = Run("MySuite.FailsWhenOptionalFieldU32HasNoValue"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(UnitTestModeTest, ProtobufOfMutatesTheProto) { auto [status, std_out, std_err] = Run("MySuite.FailsWhenI32IsSet"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("The field i32 is set!")); } TEST_F(UnitTestModeTest, ProtobufEnumEqualsLabel4) { auto [status, std_out, std_err] = Run("MySuite.FailsIfProtobufEnumEqualsLabel4"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT( std_err, HasSubstr("argument 0: fuzztest::internal::TestProtobuf::Label4")); @@ -345,7 +359,7 @@ TEST_F(UnitTestModeTest, ProtobufEnumEqualsLabel4) { TEST_F(UnitTestModeTest, WorksWithRecursiveStructs) { auto [status, std_out, std_err] = Run("MySuite.WorksWithRecursiveStructs"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); // Nullptr has multiple possible human-readable representations. EXPECT_THAT(std_err, AnyOf(HasSubstr("argument 0: LinkedList{0, 1}"), HasSubstr("argument 0: LinkedList{(nil), 1}"))); @@ -354,46 +368,46 @@ TEST_F(UnitTestModeTest, WorksWithRecursiveStructs) { TEST_F(UnitTestModeTest, WorksWithStructsWithConstructors) { auto [status, std_out, std_err] = Run("MySuite.WorksWithStructsWithConstructors"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("argument 0: HasConstructor{1, \"abc\"}")); } TEST_F(UnitTestModeTest, WorksWithStructsWithEmptyTuples) { auto [status, std_out, std_err] = Run("MySuite.WorksWithStructsWithEmptyTuples"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("argument 0: ContainsEmptyTuple{}")); } TEST_F(UnitTestModeTest, WorksWithEmptyStructs) { auto [status, std_out, std_err] = Run("MySuite.WorksWithEmptyStructs"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("argument 0: Empty{}")); } TEST_F(UnitTestModeTest, WorksWithStructsWithEmptyFields) { auto [status, std_out, std_err] = Run("MySuite.WorksWithStructsWithEmptyFields"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("argument 0: ContainsEmpty{Empty{}}")); } TEST_F(UnitTestModeTest, WorksWithEmptyInheritance) { auto [status, std_out, std_err] = Run("MySuite.WorksWithEmptyInheritance"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("argument 0: Child{0, \"abc\"}")); } TEST_F(UnitTestModeTest, ArbitraryWorksWithEmptyInheritance) { auto [status, std_out, std_err] = Run("MySuite.ArbitraryWorksWithEmptyInheritance"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("argument 0:")); } TEST_F(UnitTestModeTest, FlatMapCorrectlyPrintsValues) { auto [status, std_out, std_err] = Run("MySuite.FlatMapCorrectlyPrintsValues"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); // This is the argument to the output domain. EXPECT_THAT(std_err, HasSubstr("argument 0: {\"AAA\", \"BBB\"}")); // This is the argument to the input domain. @@ -402,24 +416,24 @@ TEST_F(UnitTestModeTest, FlatMapCorrectlyPrintsValues) { TEST_F(UnitTestModeTest, PrintsVeryLongInputsTrimmed) { auto [status, std_out, std_err] = Run("MySuite.LongInput"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("65 ...")); EXPECT_THAT(std_err, HasSubstr("A ...")); } TEST_F(UnitTestModeTest, PropertyFunctionAcceptsTupleOfItsSingleParameter) { auto [status, std_out, std_err] = Run("MySuite.UnpacksTupleOfOne"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(UnitTestModeTest, PropertyFunctionAcceptsTupleOfItsThreeParameters) { auto [status, std_out, std_err] = Run("MySuite.UnpacksTupleOfThree"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(UnitTestModeTest, PropertyFunctionAcceptsTupleContainingTuple) { auto [status, std_out, std_err] = Run("MySuite.UnpacksTupleContainingTuple"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(UnitTestModeTest, ProtoFieldsCanBeAlwaysSet) { @@ -465,7 +479,7 @@ TEST_F( TEST_F(UnitTestModeTest, DetectsRecursiveStructureIfOptionalsSetByDefault) { auto [status, std_out, std_err] = Run("MySuite.FailsIfCantInitializeProto"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("recursive fields")); } @@ -500,7 +514,7 @@ TEST_F(UnitTestModeTest, TEST_F(UnitTestModeTest, FailsWhenRepeatedFieldsSizeRangeIsInvalid) { auto [status, std_out, std_err] = Run("MySuite.FailsToInitializeIfRepeatedFieldsSizeRangeIsInvalid"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("size range is not valid")); } @@ -513,7 +527,7 @@ TEST_F(UnitTestModeTest, UsesPolicyProvidedDefaultDomainForProtos) { TEST_F(UnitTestModeTest, ChecksTypeOfProvidedDefaultDomainForProtos) { auto [status, std_out, std_err] = Run("MySuite.FailsWhenWrongDefaultProtobufDomainIsProvided"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("does not match the expected message type")); } @@ -557,7 +571,7 @@ TEST_F(UnitTestModeTest, StackLimitWorks) { /*env=*/{}, /*fuzzer_flags=*/{{"stack_limit_kb", "1000"}}); EXPECT_THAT(std_err, HasSubstr("argument 0: ")); ExpectStackLimitExceededMessage(std_err, 1024000); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(UnitTestModeTest, RssLimitFlagWorks) { @@ -566,7 +580,7 @@ TEST_F(UnitTestModeTest, RssLimitFlagWorks) { /*env=*/{}, /*fuzzer_flags=*/{{"rss_limit_mb", "1024"}}); EXPECT_THAT(std_err, HasSubstr("argument 0: ")); EXPECT_THAT(std_err, ContainsRegex(absl::StrCat("RSS limit exceeded"))); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(UnitTestModeTest, TimeLimitFlagWorks) { @@ -576,7 +590,7 @@ TEST_F(UnitTestModeTest, TimeLimitFlagWorks) { /*fuzzer_flags=*/{{"time_limit_per_input", "1s"}}); EXPECT_THAT(std_err, HasSubstr("argument 0: ")); EXPECT_THAT(std_err, ContainsRegex("Per-input timeout exceeded")); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(UnitTestModeTest, TestIsSkippedWhenRequestedInFixturePerTest) { @@ -653,12 +667,13 @@ TEST_F(GetRandomValueTest, SettingPrngSeedReproducesValue) { class GenericCommandLineInterfaceTest : public ::testing::Test { protected: RunResults RunWith( - const absl::flat_hash_map& flags, + absl::flat_hash_map flags, const absl::flat_hash_map& env = {}, absl::Duration timeout = absl::Minutes(10), absl::string_view binary = kDefaultTargetBinary, const absl::flat_hash_map& non_fuzztest_flags = {}) { + flags["print_subprocess_log"] = "true"; return RunBinary(BinaryPath(binary), RunOptions{.flags = non_fuzztest_flags, .fuzztest_flags = flags, @@ -729,14 +744,14 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, TEST_F(FuzzingModeCommandLineInterfaceTest, RunsAbortTestAndDetectsAbort) { auto [status, std_out, std_err] = RunWith({{"fuzz", "MySuite.Aborts"}}); EXPECT_THAT(std_err, HasSubstr("argument 0: ")); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(FuzzingModeCommandLineInterfaceTest, FuzzTestCanBeSelectedForFuzzingUsingSubstring) { auto [status, std_out, std_err] = RunWith({{"fuzz", "Abort"}}); EXPECT_THAT(std_err, HasSubstr("argument 0: ")); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(FuzzingModeCommandLineInterfaceTest, @@ -775,7 +790,7 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, ReproducerIsDumpedWhenEnvVarIsSet) { RunWith({{"fuzz", "MySuite.StringFast"}}, {{"FUZZTEST_REPRODUCERS_OUT_DIR", out_dir.dirname()}}); EXPECT_THAT(std_err, HasSubstr("argument 0: \"Fuzz")); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); auto replay_files = ReadFileOrDirectory(out_dir.dirname()); ASSERT_EQ(replay_files.size(), 1) << std_err; @@ -1040,6 +1055,10 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, MinimizerFindsSmallerInput) { TEST_F(FuzzingModeCommandLineInterfaceTest, FuzzerStatsArePrintedOnTermination) { +#ifdef FUZZTEST_USE_CENTIPEDE + GTEST_SKIP() << "Skip fuzzer stats test when running with Centipede because " + "stats are not printed."; +#endif auto [status, std_out, std_err] = RunWith({{"fuzz", "MySuite.PassesWithPositiveInput"}}, /*env=*/{}, @@ -1057,7 +1076,7 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, SilenceTargetWorking) { EXPECT_THAT(std_out, Not(HasSubstr("Hello World from target stdout"))); EXPECT_THAT(std_err, HasSubstr("=== Fuzzing stats")); EXPECT_THAT(std_err, Not(HasSubstr("Hello World from target stderr"))); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(FuzzingModeCommandLineInterfaceTest, NonFatalFailureAllowsMinimization) { @@ -1072,7 +1091,7 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, NonFatalFailureAllowsMinimization) { // "larger" inputs also trigger the failure. EXPECT_THAT(std_err, HasSubstr("argument 0: \"0123\"")); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(FuzzingModeCommandLineInterfaceTest, GoogleTestHasCurrentTestInfo) { @@ -1089,7 +1108,7 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, ConfiguresStackLimitByFlag) { {"stack_limit_kb", "1000"}}); EXPECT_THAT(std_err, HasSubstr("argument 0: ")); ExpectStackLimitExceededMessage(std_err, 1024000); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(FuzzingModeCommandLineInterfaceTest, @@ -1104,7 +1123,7 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, "is going to be deprecated soon. Consider switching to ", CreateFuzzTestFlag("stack_limit_kb", ""), " flag."))); ExpectStackLimitExceededMessage(std_err, 512000); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(FuzzingModeCommandLineInterfaceTest, @@ -1115,15 +1134,15 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, {{"FUZZTEST_STACK_LIMIT", "512000"}}); EXPECT_THAT(std_err, HasSubstr("argument 0: ")); ExpectStackLimitExceededMessage(std_err, 512000); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(FuzzingModeCommandLineInterfaceTest, DoesNotPrintWarningForDisabledLimitFlagsByDefault) { - auto [status, std_out, std_err] = - RunWith({{"fuzz", "MySuite.PassesWithPositiveInput"}}, - /*env=*/{}, - /*timeout=*/absl::Seconds(10)); + auto [status, std_out, std_err] = RunWith( + {{"fuzz", "MySuite.PassesWithPositiveInput"}, {"fuzz_for", "10s"}}, + /*env=*/{}, + /*timeout=*/absl::Seconds(20)); EXPECT_THAT(std_err, Not(HasSubstr("limit is specified but will be ignored"))); EXPECT_THAT(status, Eq(ExitCode(0))); @@ -1135,7 +1154,7 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, RssLimitFlagWorks) { /*env=*/{}, /*timeout=*/absl::Seconds(10)); EXPECT_THAT(std_err, HasSubstr("argument 0: ")); EXPECT_THAT(std_err, ContainsRegex(absl::StrCat("RSS limit exceeded"))); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(FuzzingModeCommandLineInterfaceTest, TimeLimitFlagWorks) { @@ -1144,7 +1163,7 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, TimeLimitFlagWorks) { /*env=*/{}); EXPECT_THAT(std_err, HasSubstr("argument 0: ")); EXPECT_THAT(std_err, ContainsRegex("Per-input timeout exceeded")); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } // TODO: b/340232436 - Once fixed, remove this test since we will no longer need @@ -1210,19 +1229,22 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, UsesCentipedeBinaryWhenEnvIsSet) { /*timeout=*/absl::Minutes(1), "testdata/unit_test_and_fuzz_tests"); EXPECT_THAT( std_err, - HasSubstr("Starting the update of the corpus database for fuzz tests")); + HasSubstr("Starting the update of the corpus database for fuzz tests")) + << std_err; EXPECT_THAT(std_err, HasSubstr("FuzzTest.AlwaysPasses")); EXPECT_THAT(status, Eq(ExitCode(0))); } -struct ExecutionModelParam { - bool multi_process; +enum class ExecutionModelParam { + kSingleBinary, + kWithCentipedeBinary, }; std::vector GetAvailableExecutionModels() { - std::vector results = {{/*multi_process=*/false}}; + std::vector results = { + ExecutionModelParam::kSingleBinary}; #ifdef FUZZTEST_USE_CENTIPEDE - results.push_back({/*multi_process=*/true}); + results.push_back(ExecutionModelParam::kWithCentipedeBinary); #endif return results; } @@ -1248,25 +1270,29 @@ class FuzzingModeFixtureTest } RunResults Run(absl::string_view test_name, int iterations) { - if (GetParam().multi_process) { - TempDir workdir; - return RunBinary( - CentipedePath(), - {.flags = {{"print_runner_log", "true"}, - {"exit_on_crash", "true"}, - {"workdir", workdir.dirname()}, - {"binary", - absl::StrCat(BinaryPath(kDefaultTargetBinary), " ", - CreateFuzzTestFlag("fuzz", test_name))}, - {"num_runs", absl::StrCat(iterations)}}, - .timeout = absl::InfiniteDuration()}); - } else { - return RunBinary( - BinaryPath(kDefaultTargetBinary), - {.fuzztest_flags = {{"fuzz", std::string(test_name)}}, - .env = {{"FUZZTEST_MAX_FUZZING_RUNS", absl::StrCat(iterations)}}, - .timeout = absl::InfiniteDuration()}); + switch (GetParam()) { + case ExecutionModelParam::kSingleBinary: + return RunBinary( + BinaryPath(kDefaultTargetBinary), + {.fuzztest_flags = {{"fuzz", std::string(test_name)}, + {"print_subprocess_log", "true"}}, + .env = {{"FUZZTEST_MAX_FUZZING_RUNS", absl::StrCat(iterations)}}, + .timeout = absl::InfiniteDuration()}); + case ExecutionModelParam::kWithCentipedeBinary: { + TempDir workdir; + return RunBinary( + CentipedePath(), + {.flags = {{"print_runner_log", "true"}, + {"exit_on_crash", "true"}, + {"workdir", workdir.dirname()}, + {"binary", + absl::StrCat(BinaryPath(kDefaultTargetBinary), " ", + CreateFuzzTestFlag("fuzz", test_name))}, + {"num_runs", absl::StrCat(iterations)}}, + .timeout = absl::InfiniteDuration()}); + } } + FUZZTEST_INTERNAL_CHECK(false, "Unsupported execution model!\n"); } }; @@ -1390,7 +1416,7 @@ class FuzzingModeCrashFindingTest // There are however env vars that we do want to propagate, which // we now need to do explicitly. env = WithTestSanitizerOptions(std::move(env)); - if (GetParam().multi_process) { + if (GetParam() == ExecutionModelParam::kWithCentipedeBinary) { TempDir workdir; return RunBinary( CentipedePath(), @@ -1405,20 +1431,10 @@ class FuzzingModeCrashFindingTest .timeout = timeout + absl::Seconds(10)}); } else { return RunBinary(BinaryPath(target_binary), - {.fuzztest_flags = {{"fuzz", std::string(test_name)}}, + {.fuzztest_flags = {{"fuzz", std::string(test_name)}, + {"fuzz_for", absl::StrCat(timeout)}}, .env = std::move(env), - .timeout = timeout}); - } - } - - void ExpectTargetAbort(TerminationStatus status, absl::string_view std_err) { - if (GetParam().multi_process) { - EXPECT_THAT(status, Ne(ExitCode(0))); - EXPECT_TRUE(RE2::PartialMatch( - std_err, absl::StrCat("Exit code\\s*:\\s*", SIGABRT))) - << std_err; - } else { - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + .timeout = timeout + absl::Seconds(10)}); } } }; @@ -1808,7 +1824,10 @@ TEST_P(FuzzingModeCrashFindingTest, TempDir out_dir; auto [status, std_out, std_err] = Run("LLVMFuzzer.TestOneInput", "testdata/llvm_fuzzer_with_custom_mutator", - /*env=*/{}, /*timeout=*/absl::Seconds(30)); + /*env=*/ + { + }, + /*timeout=*/absl::Seconds(30)); EXPECT_THAT(std_err, HasSubstr("argument 0: \"ahmfn\"")); ExpectTargetAbort(status, std_err); } diff --git a/fuzztest/BUILD b/fuzztest/BUILD index 0b5cf552..aae3363f 100644 --- a/fuzztest/BUILD +++ b/fuzztest/BUILD @@ -213,9 +213,9 @@ cc_library( deps = [ ":any", ":configuration", - ":corpus_database", ":domain_core", ":fixture_driver", + ":flag_name", ":logging", ":runtime", ":table_of_recent_compares", @@ -224,6 +224,7 @@ cc_library( "@com_google_absl//absl/memory", "@com_google_absl//absl/random", "@com_google_absl//absl/random:distributions", + "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", @@ -231,6 +232,7 @@ cc_library( "@com_google_absl//absl/time", "@com_google_absl//absl/types:span", "@com_google_fuzztest//centipede:centipede_callbacks", + "@com_google_fuzztest//centipede:centipede_default_callbacks", "@com_google_fuzztest//centipede:centipede_interface", "@com_google_fuzztest//centipede:centipede_runner_no_main", "@com_google_fuzztest//centipede:environment", @@ -623,6 +625,7 @@ cc_library( "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/time", "@com_google_absl//absl/types:span", + "@com_google_fuzztest//common:bazel", ], ) diff --git a/fuzztest/CMakeLists.txt b/fuzztest/CMakeLists.txt index a0b5ce6e..3d4ea398 100644 --- a/fuzztest/CMakeLists.txt +++ b/fuzztest/CMakeLists.txt @@ -545,6 +545,7 @@ fuzztest_cc_library( SRCS "internal/runtime.cc" DEPS + fuzztest::bazel fuzztest::configuration fuzztest::corpus_database fuzztest::coverage diff --git a/fuzztest/init_fuzztest.cc b/fuzztest/init_fuzztest.cc index cecaf31b..8707d449 100644 --- a/fuzztest/init_fuzztest.cc +++ b/fuzztest/init_fuzztest.cc @@ -97,7 +97,7 @@ FUZZTEST_DEFINE_FLAG( "Files in `crashing` directory will be used when " "--reproduce_findings_as_separate_tests flag is true. And finally, all " "files in `coverage` directory will be used when --replay_corpus flag is " - "true."); + "specified."); FUZZTEST_DEFINE_FLAG(bool, reproduce_findings_as_separate_tests, false, "When true, the selected tests replay all crashing inputs " @@ -144,6 +144,11 @@ FUZZTEST_DEFINE_FLAG( ". In the latter case, each FUZZ_TEST will run for at most (1/N)th of the " "time budget."); +FUZZTEST_DEFINE_FLAG( + std::optional, execution_id, std::nullopt, + "If set, will use the execution ID to resume or skip running on the corpus " + "database for tests that are previously run with the same execution ID."); + FUZZTEST_DEFINE_FLAG( size_t, stack_limit_kb, 128, "The soft limit of the stack size in kibibytes to abort when " @@ -163,6 +168,44 @@ FUZZTEST_DEFINE_FLAG(std::optional, jobs, std::nullopt, "The number of fuzzing jobs to run in parallel. If " "unspecified, the number of jobs is 1."); +FUZZTEST_DEFINE_FLAG( + bool, print_subprocess_log, false, + "If set, print the log of the subprocesses spawned by FuzzTest."); + +// Internal flags - not part of the user interface. +// +// These flags are meant to be set only by the parent controller process for its +// child processes. + +FUZZTEST_DEFINE_FLAG( + std::optional, internal_override_fuzz_test, std::nullopt, + "Internal-only flag - do not use directly. If set, only perform operations " + "for the exact fuzz test regardless of other flags.") + .OnUpdate([] { + FUZZTEST_INTERNAL_CHECK_PRECONDITION( + !absl::GetFlag(FUZZTEST_FLAG(internal_override_fuzz_test)) + .has_value() || + std::getenv("CENTIPEDE_RUNNER_FLAGS") != nullptr, + "must not set --" FUZZTEST_FLAG_PREFIX + "internal_override_fuzz_test directly"); + }); + +FUZZTEST_DEFINE_FLAG( + absl::Duration, internal_override_total_time_limit, + absl::InfiniteDuration(), + "Internal-only flag - do not use directly. If --" FUZZTEST_FLAG_PREFIX + "internal_override_fuzz_test is set, override the time limit set by " + "--" FUZZTEST_FLAG_PREFIX "fuzz_for / --" FUZZTEST_FLAG_PREFIX + "replay_corpus_for with --time_budget_type set to total.") + .OnUpdate([] { + FUZZTEST_INTERNAL_CHECK_PRECONDITION( + absl::GetFlag(FUZZTEST_FLAG(internal_override_total_time_limit)) == + absl::InfiniteDuration() || + std::getenv("CENTIPEDE_RUNNER_FLAGS") != nullptr, + "must not set --" FUZZTEST_FLAG_PREFIX + "internal_override_total_time_limit directly"); + }); + namespace fuzztest { std::vector ListRegisteredTests() { @@ -239,16 +282,26 @@ std::optional GetReplayCorpusTime() { internal::Configuration CreateConfigurationsFromFlags( absl::string_view binary_identifier) { - bool reproduce_findings_as_separate_tests = + const bool reproduce_findings_as_separate_tests = absl::GetFlag(FUZZTEST_FLAG(reproduce_findings_as_separate_tests)); - std::optional fuzzing_time_limit = GetFuzzingTime(); - std::optional replay_corpus_time_limit = + const std::optional fuzzing_time_limit = GetFuzzingTime(); + const std::optional replay_corpus_time_limit = GetReplayCorpusTime(); - absl::Duration time_limit = fuzzing_time_limit ? *fuzzing_time_limit - : replay_corpus_time_limit - ? *replay_corpus_time_limit - : absl::ZeroDuration(); - std::optional jobs = absl::GetFlag(FUZZTEST_FLAG(jobs)); + const std::optional override_fuzz_test = + absl::GetFlag(FUZZTEST_FLAG(internal_override_fuzz_test)); + const bool replay_coverage_inputs = + fuzzing_time_limit.has_value() || replay_corpus_time_limit.has_value(); + const absl::Duration time_limit = + override_fuzz_test.has_value() + ? absl::GetFlag(FUZZTEST_FLAG(internal_override_total_time_limit)) + : fuzzing_time_limit.has_value() ? *fuzzing_time_limit + : replay_corpus_time_limit.has_value() ? *replay_corpus_time_limit + : absl::ZeroDuration(); + const internal::TimeBudgetType time_budget_type = + override_fuzz_test.has_value() + ? internal::TimeBudgetType::kTotal + : absl::GetFlag(FUZZTEST_FLAG(time_budget_type)); + const std::optional jobs = absl::GetFlag(FUZZTEST_FLAG(jobs)); FUZZTEST_INTERNAL_CHECK(!jobs.has_value() || *jobs > 0, "If specified, --", FUZZTEST_FLAG(jobs).Name(), " must be positive."); return internal::Configuration{ @@ -257,13 +310,15 @@ internal::Configuration CreateConfigurationsFromFlags( std::string(binary_identifier), /*fuzz_tests=*/ListRegisteredTests(), /*fuzz_tests_in_current_shard=*/ListRegisteredTests(), - reproduce_findings_as_separate_tests, - /*only_replay_corpus=*/ + reproduce_findings_as_separate_tests, replay_coverage_inputs, + /*only_replay=*/ replay_corpus_time_limit.has_value(), + absl::GetFlag(FUZZTEST_FLAG(execution_id)), + absl::GetFlag(FUZZTEST_FLAG(print_subprocess_log)), /*stack_limit=*/absl::GetFlag(FUZZTEST_FLAG(stack_limit_kb)) * 1024, /*rss_limit=*/absl::GetFlag(FUZZTEST_FLAG(rss_limit_mb)) * 1024 * 1024, absl::GetFlag(FUZZTEST_FLAG(time_limit_per_input)), time_limit, - absl::GetFlag(FUZZTEST_FLAG(time_budget_type)), jobs.value_or(0)}; + time_budget_type, jobs.value_or(0)}; } } // namespace @@ -281,6 +336,7 @@ void RunSpecifiedFuzzTest(std::string_view name, std::string_view binary_id) { } void InitFuzzTest(int* argc, char*** argv, std::string_view binary_id) { + auto& runtime = internal::Runtime::instance(); const bool is_listing = absl::GetFlag(FUZZTEST_FLAG(list_fuzz_tests)); if (is_listing) { for (const auto& name : ListRegisteredTests()) { @@ -297,12 +353,26 @@ void InitFuzzTest(int* argc, char*** argv, std::string_view binary_id) { const auto test_to_fuzz = absl::GetFlag(FUZZTEST_FLAG(fuzz)); const auto test_to_replay_corpus = absl::GetFlag(FUZZTEST_FLAG(replay_corpus)); - const auto specified_test = - test_to_fuzz != kUnspecified ? test_to_fuzz : test_to_replay_corpus; - const bool is_test_specified = specified_test != kUnspecified; - if (is_test_specified) { + const auto specified_test = []() -> std::optional { + if (auto internal_selected_test = + absl::GetFlag(FUZZTEST_FLAG(internal_override_fuzz_test)); + internal_selected_test.has_value()) { + return internal_selected_test; + } + if (auto test_to_fuzz = absl::GetFlag(FUZZTEST_FLAG(fuzz)); + test_to_fuzz != kUnspecified) { + return test_to_fuzz; + } + if (auto test_to_replay_corpus = + absl::GetFlag(FUZZTEST_FLAG(replay_corpus)); + test_to_replay_corpus != kUnspecified) { + return test_to_replay_corpus; + } + return std::nullopt; + }(); + if (specified_test.has_value()) { const std::string matching_fuzz_test = - GetMatchingFuzzTestOrExit(specified_test); + GetMatchingFuzzTestOrExit(*specified_test); // Delegate the test to GoogleTest. GTEST_FLAG_SET(filter, matching_fuzz_test); } @@ -318,7 +388,7 @@ void InitFuzzTest(int* argc, char*** argv, std::string_view binary_id) { const bool is_fuzzing_or_replaying = (fuzzing_time_limit || replay_corpus_time_limit); - if (is_fuzzing_or_replaying && !is_test_specified) { + if (is_fuzzing_or_replaying && !specified_test.has_value()) { absl::flat_hash_set fuzz_tests = { configuration.fuzz_tests.begin(), configuration.fuzz_tests.end()}; std::vector non_fuzz_tests; @@ -343,13 +413,10 @@ void InitFuzzTest(int* argc, char*** argv, std::string_view binary_id) { GTEST_FLAG_SET(filter, filter); } } - const bool is_runner_mode = std::getenv("CENTIPEDE_RUNNER_FLAGS") != nullptr; - const bool is_fuzzing_mode = (is_runner_mode && is_fuzzing_or_replaying) || - fuzzing_time_limit.has_value(); const RunMode run_mode = - is_fuzzing_mode ? RunMode::kFuzz : RunMode::kUnitTest; + fuzzing_time_limit.has_value() ? RunMode::kFuzz : RunMode::kUnitTest; // TODO(b/307513669): Use the Configuration class instead of Runtime. - internal::Runtime::instance().SetRunMode(run_mode); + runtime.SetRunMode(run_mode); } void ParseAbslFlags(int argc, char** argv) { diff --git a/fuzztest/internal/centipede_adaptor.cc b/fuzztest/internal/centipede_adaptor.cc index f35e7e80..e69199e5 100644 --- a/fuzztest/internal/centipede_adaptor.cc +++ b/fuzztest/internal/centipede_adaptor.cc @@ -15,6 +15,13 @@ #include "./fuzztest/internal/centipede_adaptor.h" #include +#ifdef __APPLE__ +#include +#else // __APPLE__ +#include // ARG_MAX +#endif // __APPLE__ +#include +#include #include #include @@ -41,16 +48,19 @@ #include "absl/memory/memory.h" #include "absl/random/distributions.h" #include "absl/random/random.h" +#include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/match.h" #include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" +#include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" #include "absl/time/clock.h" #include "absl/time/time.h" #include "absl/types/span.h" #include "./centipede/centipede_callbacks.h" +#include "./centipede/centipede_default_callbacks.h" #include "./centipede/centipede_interface.h" #include "./centipede/environment.h" #include "./centipede/mutation_input.h" @@ -62,9 +72,9 @@ #include "./common/defs.h" #include "./fuzztest/internal/any.h" #include "./fuzztest/internal/configuration.h" -#include "./fuzztest/internal/corpus_database.h" #include "./fuzztest/internal/domains/domain.h" #include "./fuzztest/internal/fixture_driver.h" +#include "./fuzztest/internal/flag_name.h" #include "./fuzztest/internal/logging.h" #include "./fuzztest/internal/runtime.h" #include "./fuzztest/internal/table_of_recent_compares.h" @@ -98,6 +108,87 @@ class TempDir { std::string path_; }; +absl::StatusOr> GetProcessArgs() { + std::vector results; +#if defined(__APPLE__) + // Reference: + // https://chromium.googlesource.com/crashpad/crashpad/+/360e441c53ab4191a6fd2472cc57c3343a2f6944/util/posix/process_util_mac.cc + char procargs[ARG_MAX]; + size_t procargs_size = sizeof(procargs); + int mib[] = {CTL_KERN, KERN_PROCARGS2, getpid()}; + const int rv = sysctl(mib, sizeof(mib) / sizeof(mib[0]), procargs, + &procargs_size, nullptr, 0); + if (rv != 0) { + return absl::InternalError( + "GetEnv: sysctl({CTK_KERN, KERN_PROCARGS2, ...}) failed"); + } + if (procargs_size < sizeof(int)) { + return absl::InternalError("GetEnv: procargs_size too small"); + } + int argc = 0; + std::memcpy(&argc, &procargs[0], sizeof(argc)); + size_t start_pos = sizeof(argc); + // Find the end of the executable path. + while (start_pos < procargs_size && procargs[start_pos] != 0) ++start_pos; + if (start_pos == procargs_size) { + return absl::NotFoundError("nothing after executable path"); + } + // Find the beginning of the string area. + while (start_pos < procargs_size && procargs[start_pos] == 0) ++start_pos; + if (start_pos == procargs_size) { + return absl::NotFoundError("nothing after executable path"); + } + // Get the first argc c-strings without exceeding the boundary. + for (int i = 0; i < argc; ++i) { + const size_t current_argv_pos = start_pos; + while (start_pos < procargs_size && procargs[start_pos] != 0) ++start_pos; + if (start_pos == procargs_size) { + return absl::InternalError("incomplete argv list in the procargs"); + } + results.emplace_back(&procargs[current_argv_pos], + start_pos - current_argv_pos); + ++start_pos; + } + return result; +#elif defined(__linux__) + const int fd = open("/proc/self/cmdline", O_RDONLY); + if (fd < 0) { + return absl::InternalError( + absl::StrCat("failed opening /proc/self/cmdline: ", strerror(errno))); + } + std::string args; + while (true) { + char buf[4096]; + const ssize_t read_size = read(fd, buf, sizeof(buf)); + if (read_size == 0) break; + if (read_size < 0) { + return absl::InternalError( + absl::StrCat("failed reading /proc/self/cmdline: ", strerror(errno))); + } + args.append(buf, read_size); + } + if (close(fd) != 0) { + return absl::InternalError( + absl::StrCat("failed closing /proc/self/cmdline: ", strerror(errno))); + } + size_t start_pos = 0; + while (start_pos < args.size()) { + const size_t current_argv_pos = start_pos; + while (start_pos < args.size() && args[start_pos] != 0) ++start_pos; + results.emplace_back(&args[current_argv_pos], start_pos - current_argv_pos); + ++start_pos; + } + return results; +#else // !defined(__APPLE__) && !defined(__linux) + return absl::UnimplementedError( + absl::StrCat(__func__, "() not implemented on the platform")); +#endif +} + +std::string ShellEscape(absl::string_view str) { + return absl::StrCat("'", absl::StrReplaceAll(str, {{"'", "'\\''"}}), "'"); +} + // TODO(xinhaoyuan): Consider passing rng seeds from the engine. std::seed_seq GetRandomSeed() { const size_t seed = time(nullptr) + getpid() + @@ -118,35 +209,102 @@ centipede::Environment CreateDefaultCentipedeEnvironment() { return env; } -centipede::Environment CreateCentipedeEnvironmentFromFuzzTestFlags( +centipede::Environment CreateCentipedeEnvironmentFromConfiguration( const Configuration& configuration, absl::string_view workdir, - absl::string_view test_name) { + absl::string_view test_name, RunMode run_mode) { centipede::Environment env = CreateDefaultCentipedeEnvironment(); + constexpr absl::Duration kUnitTestDefaultDuration = absl::Seconds(3); + env.fuzztest_single_test_mode = true; + env.populate_binary_info = false; + const auto args = GetProcessArgs(); + FUZZTEST_INTERNAL_CHECK( + args.ok(), + absl::StrCat("failed to get the original process args: ", args.status())); + env.binary.clear(); + for (const auto& arg : *args) { + // We need shell escaping, because env.binary will be passed to system(), + // which uses the default shell. + absl::StrAppend(&env.binary, env.binary.empty() ? "" : " ", + ShellEscape(arg)); + } + absl::StrAppend( + &env.binary, + " --" FUZZTEST_FLAG_PREFIX "internal_override_fuzz_test=", test_name); + absl::Duration total_time_limit = configuration.GetTimeLimitPerTest(); + // TODO(xinhaoyuan): Consider using unset optional duration instead of zero + // duration as the special value. + if (total_time_limit == absl::ZeroDuration() && + run_mode == RunMode::kUnitTest) { + total_time_limit = kUnitTestDefaultDuration; + } + absl::StrAppend(&env.binary, + " --" FUZZTEST_FLAG_PREFIX + "internal_override_total_time_limit=", + total_time_limit); + env.coverage_binary = (*args)[0]; + env.exit_on_crash = + // Do shallow testing when running in unit-test mode unless we are replay + // coverage inputs. + (run_mode == RunMode::kUnitTest && + !configuration.replay_coverage_inputs) || + // When not using a corpus database, keep the same behavior as the legacy + // single-process mode. + configuration.corpus_database.empty() || + // No need to keep running when replaying crashing input. + configuration.crashing_input_to_reproduce.has_value(); + env.print_runner_log = configuration.print_subprocess_log; env.workdir = workdir; - env.exit_on_crash = true; - // Populating the PC table in single-process mode is not implemented. - env.require_pc_table = false; - const auto time_limit_per_test = configuration.GetTimeLimitPerTest(); - if (time_limit_per_test != absl::InfiniteDuration()) { - absl::FPrintF(GetStderr(), "[.] Fuzzing timeout set to: %s\n", - absl::FormatDuration(time_limit_per_test)); - env.stop_at = absl::Now() + time_limit_per_test; - } - env.first_corpus_dir_output_only = true; - if (const char* corpus_out_dir_chars = getenv("FUZZTEST_TESTSUITE_OUT_DIR")) { - env.corpus_dir.push_back(corpus_out_dir_chars); + if (configuration.corpus_database.empty()) { + if (total_time_limit != absl::InfiniteDuration()) { + absl::FPrintF(GetStderr(), "[.] Fuzzing timeout set to: %s\n", + absl::FormatDuration(total_time_limit)); + env.stop_at = absl::Now() + total_time_limit; + } + env.first_corpus_dir_output_only = true; + if (const char* corpus_out_dir_chars = + std::getenv("FUZZTEST_TESTSUITE_OUT_DIR")) { + env.corpus_dir.push_back(corpus_out_dir_chars); + } else { + env.corpus_dir.push_back(""); + } + if (const char* corpus_in_dir_chars = + std::getenv("FUZZTEST_TESTSUITE_IN_DIR")) { + env.corpus_dir.push_back(corpus_in_dir_chars); + } + if (const char* max_fuzzing_runs = + std::getenv("FUZZTEST_MAX_FUZZING_RUNS")) { + if (!absl::SimpleAtoi(max_fuzzing_runs, &env.num_runs)) { + absl::FPrintF( + GetStderr(), + "[!] Cannot parse env FUZZTEST_MAX_FUZZING_RUNS=%s - will " + "not limit fuzzing runs.\n", + max_fuzzing_runs); + env.num_runs = std::numeric_limits::max(); + } + } } else { - env.corpus_dir.push_back(""); - } - if (const char* corpus_in_dir_chars = getenv("FUZZTEST_TESTSUITE_IN_DIR")) - env.corpus_dir.push_back(corpus_in_dir_chars); - if (const char* max_fuzzing_runs = getenv("FUZZTEST_MAX_FUZZING_RUNS")) { - if (!absl::SimpleAtoi(max_fuzzing_runs, &env.num_runs)) { + // Not setting env.stop_at since current update_corpus logic in Centipede + // would propagate that. + if (std::getenv("FUZZTEST_TESTSUITE_OUT_DIR")) { + absl::FPrintF(GetStderr(), + "[!] Ignoring FUZZTEST_TESTSUITE_OUT_DIR when the corpus " + "database is set.\n"); + } + if (std::getenv("FUZZTEST_TESTSUITE_IN_DIR")) { + absl::FPrintF(GetStderr(), + "[!] Ignoring FUZZTEST_TESTSUITE_IN_DIR when the corpus " + "database is set.\n"); + } + if (std::getenv("FUZZTEST_MINIMIZE_TESTSUITE_DIR")) { + absl::FPrintF(GetStderr(), + "[!] Ignoring FUZZTEST_MINIMIZE_TESTSUITE_DIR when the " + "corpus database is set.\n"); + } + if (const char* max_fuzzing_runs = + std::getenv("FUZZTEST_MAX_FUZZING_RUNS")) { absl::FPrintF(GetStderr(), - "[!] Cannot parse env FUZZTEST_MAX_FUZZING_RUNS=%s - will " - "not limit fuzzing runs.\n", - max_fuzzing_runs); - env.num_runs = std::numeric_limits::max(); + "[!] Ignoring FUZZTEST_MAX_FUZZING_RUNS when the " + "corpus database is set.\n"); } } return env; @@ -186,13 +344,6 @@ class CentipedeAdaptorRunnerCallbacks : public centipede::RunnerCallbacks { std::function seed_callback) override { std::vector seeds = fuzzer_impl_.fixture_driver_->GetSeeds(); - CorpusDatabase corpus_database(configuration_); - fuzzer_impl_.ForEachInput( - corpus_database.GetCoverageInputsIfAny(fuzzer_impl_.test_.full_name()), - [&](absl::string_view /*file_path*/, std::optional /*blob_idx*/, - FuzzTestFuzzerImpl::Input input) { - seeds.push_back(std::move(input.args)); - }); constexpr int kInitialValuesInSeeds = 32; for (int i = 0; i < kInitialValuesInSeeds; ++i) { seeds.push_back(fuzzer_impl_.params_domain_.Init(prng_)); @@ -237,7 +388,7 @@ class CentipedeAdaptorRunnerCallbacks : public centipede::RunnerCallbacks { size_t num_mutants, std::function new_mutant_callback) override { if (inputs.empty()) return false; - SetMetadata(inputs[0].metadata); + if (runtime_.run_mode() == RunMode::kFuzz) SetMetadata(inputs[0].metadata); for (size_t i = 0; i < num_mutants; ++i) { const auto choice = absl::Uniform(prng_, 0, 1); std::string mutant_data; @@ -315,124 +466,6 @@ class CentipedeAdaptorRunnerCallbacks : public centipede::RunnerCallbacks { namespace { -class CentipedeAdaptorEngineCallbacks : public centipede::CentipedeCallbacks { - public: - CentipedeAdaptorEngineCallbacks(const centipede::Environment& env, - Runtime* runtime, - FuzzTestFuzzerImpl* fuzzer_impl, - const Configuration* configuration) - : centipede::CentipedeCallbacks(env), - runtime_(*runtime), - runner_callbacks_(runtime, fuzzer_impl, configuration), - batch_result_buffer_size_(env.shmem_size_mb * 1024 * 1024), - batch_result_buffer_(nullptr) {} - - ~CentipedeAdaptorEngineCallbacks() { - if (batch_result_buffer_ != nullptr) - munmap(batch_result_buffer_, batch_result_buffer_size_); - } - - bool Execute(std::string_view binary, - const std::vector& inputs, - centipede::BatchResult& batch_result) override { - // Execute the test in-process. - batch_result.ClearAndResize(inputs.size()); - size_t buffer_offset = 0; - if (batch_result_buffer_ == nullptr) { - // Use mmap which allocates memory on demand to reduce sanitizer overhead. - batch_result_buffer_ = static_cast( - mmap(nullptr, batch_result_buffer_size_, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); - FUZZTEST_INTERNAL_CHECK( - batch_result_buffer_ != MAP_FAILED, - "Cannot mmap anonymous memory for batch result buffer"); - } - CentipedeBeginExecutionBatch(); - for (const auto& input : inputs) { - if (runtime_.termination_requested()) break; - if (buffer_offset >= batch_result_buffer_size_) break; - runner_callbacks_.Execute(input); - buffer_offset += CentipedeGetExecutionResult( - batch_result_buffer_ + buffer_offset, - batch_result_buffer_size_ - buffer_offset); - } - CentipedeEndExecutionBatch(); - if (buffer_offset > 0) { - centipede::BlobSequence batch_result_blobseq(batch_result_buffer_, - buffer_offset); - batch_result.Read(batch_result_blobseq); - } - if (runtime_.termination_requested() && !centipede::ShouldStop()) { - absl::FPrintF(GetStderr(), "[.] Early termination requested.\n"); - centipede::RequestEarlyStop(0); - } - return true; - } - - size_t GetSeeds(size_t num_seeds, - std::vector& seeds) override { - seeds.clear(); - size_t num_avail_seeds = 0; - runner_callbacks_.GetSeeds([&](centipede::ByteSpan seed) { - ++num_avail_seeds; - if (seeds.size() < num_seeds) { - seeds.emplace_back(seed.begin(), seed.end()); - } - }); - return num_avail_seeds; - } - - absl::StatusOr GetSerializedTargetConfig() override { - return runner_callbacks_.GetSerializedTargetConfig(); - } - - void Mutate(const std::vector& inputs, - size_t num_mutants, - std::vector& mutants) override { - mutants.clear(); - runner_callbacks_.Mutate( - inputs, num_mutants, [&](centipede::ByteSpan mutant) { - mutants.emplace_back(mutant.begin(), mutant.end()); - }); - if (runtime_.termination_requested() && !centipede::ShouldStop()) { - absl::FPrintF(GetStderr(), "[.] Early termination requested.\n"); - centipede::RequestEarlyStop(0); - } - } - - private: - Runtime& runtime_; - CentipedeAdaptorRunnerCallbacks runner_callbacks_; - size_t batch_result_buffer_size_; - uint8_t* batch_result_buffer_; -}; - -class CentipedeAdaptorEngineCallbacksFactory - : public centipede::CentipedeCallbacksFactory { - public: - CentipedeAdaptorEngineCallbacksFactory(Runtime* runtime, - FuzzTestFuzzerImpl* fuzzer_impl, - const Configuration* configuration) - : runtime_(runtime), - fuzzer_impl_(fuzzer_impl), - configuration_(configuration) {} - - centipede::CentipedeCallbacks* create( - const centipede::Environment& env) override { - return new CentipedeAdaptorEngineCallbacks(env, runtime_, fuzzer_impl_, - configuration_); - } - - void destroy(centipede::CentipedeCallbacks* callbacks) override { - delete callbacks; - } - - private: - Runtime* runtime_; - FuzzTestFuzzerImpl* fuzzer_impl_; - const Configuration* configuration_; -}; - void PopulateTestLimitsToCentipedeRunner(const Configuration& configuration) { if (const size_t stack_limit = GetStackLimitFromEnvOrConfiguration(configuration); @@ -513,7 +546,7 @@ class CentipedeFixtureDriver : public UntypedFixtureDriver { private: const Configuration* configuration_ = nullptr; Runtime& runtime_; - const bool runner_mode = getenv("CENTIPEDE_RUNNER_FLAGS") != nullptr; + const bool runner_mode = std::getenv("CENTIPEDE_RUNNER_FLAGS") != nullptr; std::unique_ptr orig_fixture_driver_; }; @@ -529,27 +562,43 @@ CentipedeFuzzerAdaptor::CentipedeFuzzerAdaptor( bool CentipedeFuzzerAdaptor::RunInUnitTestMode( const Configuration& configuration) { - centipede_fixture_driver_->set_configuration(&configuration); - CentipedeBeginExecutionBatch(); - fuzzer_impl_.RunInUnitTestMode(configuration); - CentipedeEndExecutionBatch(); - return true; + return Run(/*argc=*/nullptr, /*argv=*/nullptr, RunMode::kUnitTest, + configuration); } bool CentipedeFuzzerAdaptor::RunInFuzzingMode( int* argc, char*** argv, const Configuration& configuration) { + return Run(argc, argv, RunMode::kFuzz, configuration); +} + +// TODO(xinhaoyuan): Consider merging `mode` into `configuration`. +bool CentipedeFuzzerAdaptor::Run(int* argc, char*** argv, RunMode mode, + const Configuration& configuration) { centipede_fixture_driver_->set_configuration(&configuration); - runtime_.SetRunMode(RunMode::kFuzz); - runtime_.SetSkippingRequested(false); - runtime_.SetCurrentTest(&test_, &configuration); - if (IsSilenceTargetEnabled()) SilenceTargetStdoutAndStderr(); - runtime_.EnableReporter(&fuzzer_impl_.stats_, [] { return absl::Now(); }); - fuzzer_impl_.fixture_driver_->SetUpFuzzTest(); - bool print_final_stats = true; // When the CENTIPEDE_RUNNER_FLAGS env var exists, the current process is // considered a child process spawned by the Centipede binary as the runner, // and we should not run CentipedeMain in this process. - const bool runner_mode = getenv("CENTIPEDE_RUNNER_FLAGS"); + const bool runner_mode = std::getenv("CENTIPEDE_RUNNER_FLAGS"); + const bool is_running_property_function_in_this_process = + runner_mode || configuration.crashing_input_to_reproduce.has_value() || + std::getenv("FUZZTEST_REPLAY") || + std::getenv("FUZZTEST_MINIMIZE_REPRODUCER"); + if (!is_running_property_function_in_this_process && + runtime_.termination_requested()) { + absl::FPrintF(GetStderr(), + "[.] Skipping %s since termination was requested.\n", + test_.full_name()); + runtime_.SetSkippingRequested(true); + return true; + } + runtime_.SetRunMode(mode); + runtime_.SetSkippingRequested(false); + runtime_.SetCurrentTest(&test_, &configuration); + if (is_running_property_function_in_this_process) { + if (IsSilenceTargetEnabled()) SilenceTargetStdoutAndStderr(); + runtime_.EnableReporter(&fuzzer_impl_.stats_, [] { return absl::Now(); }); + } + fuzzer_impl_.fixture_driver_->SetUpFuzzTest(); const int result = ([&]() { if (runtime_.skipping_requested()) { absl::FPrintF(GetStderr(), @@ -560,24 +609,31 @@ bool CentipedeFuzzerAdaptor::RunInFuzzingMode( if (runner_mode) { CentipedeAdaptorRunnerCallbacks runner_callbacks(&runtime_, &fuzzer_impl_, &configuration); - print_final_stats = false; - return centipede::RunnerMain(argc != nullptr ? *argc : 0, - argv != nullptr ? *argv : nullptr, + static char fake_argv0[] = "fake_argv"; + static char* fake_argv[] = {fake_argv0, nullptr}; + return centipede::RunnerMain(argc != nullptr ? *argc : 1, + argv != nullptr ? *argv : fake_argv, runner_callbacks); } // Centipede engine does not support replay and reproducer minimization // (within the single process). So use the existing fuzztest implementation. // This is fine because it does not require coverage instrumentation. if (fuzzer_impl_.ReplayInputsIfAvailable(configuration)) return 0; + // `ReplayInputsIfAvailable` overwrites the run mode - revert it back. + runtime_.SetRunMode(mode); // Run as the fuzzing engine. - TempDir workdir("/tmp/fuzztest-workdir-"); - const auto env = CreateCentipedeEnvironmentFromFuzzTestFlags( - configuration, workdir.path(), test_.full_name()); - CentipedeAdaptorEngineCallbacksFactory factory(&runtime_, &fuzzer_impl_, - &configuration); + std::unique_ptr workdir; + if (configuration.corpus_database.empty() || mode == RunMode::kUnitTest) + workdir = std::make_unique("/tmp/fuzztest-workdir-"); + const std::string workdir_path = workdir ? workdir->path() : ""; + const auto env = CreateCentipedeEnvironmentFromConfiguration( + configuration, workdir_path, test_.full_name(), mode); + centipede::DefaultCallbacksFactory + factory; if (const char* minimize_dir_chars = - std::getenv("FUZZTEST_MINIMIZE_TESTSUITE_DIR")) { - print_final_stats = false; + std::getenv("FUZZTEST_MINIMIZE_TESTSUITE_DIR"); + configuration.corpus_database.empty() && + minimize_dir_chars != nullptr) { const std::string minimize_dir = minimize_dir_chars; const char* corpus_out_dir_chars = std::getenv("FUZZTEST_TESTSUITE_OUT_DIR"); @@ -629,11 +685,6 @@ bool CentipedeFuzzerAdaptor::RunInFuzzingMode( return centipede::CentipedeMain(env, factory); })(); fuzzer_impl_.fixture_driver_->TearDownFuzzTest(); - if (print_final_stats) { - absl::FPrintF(GetStderr(), "\n[.] Fuzzing was terminated.\n"); - runtime_.PrintFinalStatsOnDefaultSink(); - absl::FPrintF(GetStderr(), "\n"); - } return result == 0; } @@ -671,10 +722,11 @@ class CentipedeCallbacksForRunnerFlagsExtraction } // namespace extern "C" const char* CentipedeGetRunnerFlags() { - if (const char* runner_flags_env = getenv("CENTIPEDE_RUNNER_FLAGS")) { + if (const char* runner_flags_env = std::getenv("CENTIPEDE_RUNNER_FLAGS")) { // Runner mode. Use the existing flags. return strdup(runner_flags_env); } + // Set the runner flags according to the FuzzTest default environment. const auto env = fuzztest::internal::CreateDefaultCentipedeEnvironment(); CentipedeCallbacksForRunnerFlagsExtraction callbacks(env); diff --git a/fuzztest/internal/centipede_adaptor.h b/fuzztest/internal/centipede_adaptor.h index 6ca4c815..295574fb 100644 --- a/fuzztest/internal/centipede_adaptor.h +++ b/fuzztest/internal/centipede_adaptor.h @@ -35,6 +35,9 @@ class CentipedeFuzzerAdaptor : public FuzzTestFuzzer { const Configuration& configuration) override; private: + bool Run(int* argc, char*** argv, RunMode mode, + const Configuration& configuration); + Runtime& runtime_ = Runtime::instance(); const FuzzTest& test_; CentipedeFixtureDriver* centipede_fixture_driver_; diff --git a/fuzztest/internal/configuration.cc b/fuzztest/internal/configuration.cc index 31f35574..3dea32f9 100644 --- a/fuzztest/internal/configuration.cc +++ b/fuzztest/internal/configuration.cc @@ -206,10 +206,12 @@ std::string Configuration::Serialize() const { SpaceFor(binary_identifier) + SpaceFor(fuzz_tests) + SpaceFor(fuzz_tests_in_current_shard) + SpaceFor(reproduce_findings_as_separate_tests) + - SpaceFor(only_replay_corpus) + SpaceFor(stack_limit) + - SpaceFor(rss_limit) + SpaceFor(time_limit_per_input_str) + - SpaceFor(time_limit_str) + SpaceFor(time_budget_type_str) + - SpaceFor(jobs) + SpaceFor(crashing_input_to_reproduce) + + SpaceFor(replay_coverage_inputs) + SpaceFor(only_replay) + + SpaceFor(execution_id) + SpaceFor(print_subprocess_log) + + SpaceFor(stack_limit) + SpaceFor(rss_limit) + + SpaceFor(time_limit_per_input_str) + SpaceFor(time_limit_str) + + SpaceFor(time_budget_type_str) + SpaceFor(jobs) + + SpaceFor(crashing_input_to_reproduce) + SpaceFor(reproduction_command_template)); size_t offset = 0; offset = WriteString(out, offset, corpus_database); @@ -218,7 +220,10 @@ std::string Configuration::Serialize() const { offset = WriteVectorOfStrings(out, offset, fuzz_tests); offset = WriteVectorOfStrings(out, offset, fuzz_tests_in_current_shard); offset = WriteIntegral(out, offset, reproduce_findings_as_separate_tests); - offset = WriteIntegral(out, offset, only_replay_corpus); + offset = WriteIntegral(out, offset, replay_coverage_inputs); + offset = WriteIntegral(out, offset, only_replay); + offset = WriteOptionalString(out, offset, execution_id); + offset = WriteIntegral(out, offset, print_subprocess_log); offset = WriteIntegral(out, offset, stack_limit); offset = WriteIntegral(out, offset, rss_limit); offset = WriteString(out, offset, time_limit_per_input_str); @@ -242,7 +247,10 @@ absl::StatusOr Configuration::Deserialize( ConsumeVectorOfStrings(serialized)); ASSIGN_OR_RETURN(reproduce_findings_as_separate_tests, Consume(serialized)); - ASSIGN_OR_RETURN(only_replay_corpus, Consume(serialized)); + ASSIGN_OR_RETURN(replay_coverage_inputs, Consume(serialized)); + ASSIGN_OR_RETURN(only_replay, Consume(serialized)); + ASSIGN_OR_RETURN(execution_id, ConsumeOptionalString(serialized)); + ASSIGN_OR_RETURN(print_subprocess_log, Consume(serialized)); ASSIGN_OR_RETURN(stack_limit, Consume(serialized)); ASSIGN_OR_RETURN(rss_limit, Consume(serialized)); ASSIGN_OR_RETURN(time_limit_per_input_str, ConsumeString(serialized)); @@ -268,7 +276,10 @@ absl::StatusOr Configuration::Deserialize( *std::move(fuzz_tests), *std::move(fuzz_tests_in_current_shard), *reproduce_findings_as_separate_tests, - *only_replay_corpus, + *replay_coverage_inputs, + *only_replay, + *std::move(execution_id), + *print_subprocess_log, *stack_limit, *rss_limit, *time_limit_per_input, diff --git a/fuzztest/internal/configuration.h b/fuzztest/internal/configuration.h index cccf5387..cfe82b9c 100644 --- a/fuzztest/internal/configuration.h +++ b/fuzztest/internal/configuration.h @@ -59,8 +59,23 @@ struct Configuration { // Generate separate TESTs that replay crashing inputs for the selected fuzz // tests. bool reproduce_findings_as_separate_tests = false; - // Do not fuzz, only replay the corpus. - bool only_replay_corpus = false; + // When working on a corpus database, a few steps can be performed for each + // test: + // 1. Replaying inputs from the database, by default the regression inputs + // will be used. + // 2. Fuzzing with generated inputs, some of which trigger crashes. + // 3. Updating the database with the coverage-increasing and crashing + // inputs. + // + // If set, coverage inputs are included for replaying. + bool replay_coverage_inputs = false; + // If set, further steps are skipped after replaying. + bool only_replay = false; + // If set, will be used when working on a corpus database to resume + // the progress in case the execution got interrupted. + std::optional execution_id; + // If set, print log from subprocesses spawned by FuzzTest. + bool print_subprocess_log = false; // Stack limit in bytes. size_t stack_limit = 128 * 1024; diff --git a/fuzztest/internal/configuration_test.cc b/fuzztest/internal/configuration_test.cc index 4ce02a1a..41463791 100644 --- a/fuzztest/internal/configuration_test.cc +++ b/fuzztest/internal/configuration_test.cc @@ -21,7 +21,10 @@ MATCHER_P(IsOkAndEquals, config, "") { other->fuzz_tests_in_current_shard && config.reproduce_findings_as_separate_tests == other->reproduce_findings_as_separate_tests && - config.only_replay_corpus == other->only_replay_corpus && + config.replay_coverage_inputs == other->replay_coverage_inputs && + config.only_replay == other->only_replay && + config.execution_id == other->execution_id && + config.print_subprocess_log == other->print_subprocess_log && config.stack_limit == other->stack_limit && config.rss_limit == other->rss_limit && config.time_limit_per_input == other->time_limit_per_input && @@ -42,7 +45,10 @@ TEST(ConfigurationTest, /*fuzz_tests=*/{}, /*fuzz_tests_in_current_shard=*/{}, /*reproduce_findings_as_separate_tests=*/true, - /*only_replay_corpus=*/true, + /*replay_coverage_inputs=*/true, + /*only_replay=*/true, + /*execution_id*/ "execution_id", + /*print_subprocess_log=*/true, /*stack_limit=*/100, /*rss_limit=*/200, /*time_limit_per_input=*/absl::Seconds(42), @@ -64,7 +70,10 @@ TEST(ConfigurationTest, {"FuzzTest1", "FuzzTest2"}, {"FuzzTest1"}, /*reproduce_findings_as_separate_tests=*/true, - /*only_replay_corpus=*/true, + /*replay_coverage_inputs=*/true, + /*only_replay=*/true, + /*execution_id*/ "execution_id", + /*print_subprocess_log=*/true, /*stack_limit=*/100, /*rss_limit=*/200, /*time_limit_per_input=*/absl::Seconds(42), diff --git a/fuzztest/internal/runtime.cc b/fuzztest/internal/runtime.cc index fae36a34..1f57fba4 100644 --- a/fuzztest/internal/runtime.cc +++ b/fuzztest/internal/runtime.cc @@ -20,6 +20,7 @@ #endif #include +#include #include #include #include @@ -54,6 +55,7 @@ #include "absl/time/clock.h" #include "absl/time/time.h" #include "absl/types/span.h" +#include "./common/bazel.h" #include "./fuzztest/internal/configuration.h" #include "./fuzztest/internal/corpus_database.h" #include "./fuzztest/internal/coverage.h" @@ -396,6 +398,29 @@ void Runtime::CheckWatchdogLimits() { #endif } +void Runtime::SetCurrentTest(const FuzzTest* test, + const Configuration* configuration) { + CHECK((test != nullptr) == (configuration != nullptr)); + current_test_ = test; + current_configuration_ = configuration; + if (configuration == nullptr) return; + if (const auto test_time_limit = configuration->GetTimeLimitPerTest(); + test_time_limit < absl::InfiniteDuration()) { + const absl::Status has_enough_time = + centipede::CheckBazelHasEnoughTimeToRunTest( + creation_time_, test_time_limit, test_counter_, + configuration->fuzz_tests.size()); + if (!has_enough_time.ok()) { + absl::FPrintF(GetStderr(), + "[!] Not enough time for running the fuzz test %s for " + "%s. Still running the test anyway. Error: %s\n", + test->full_name(), absl::FormatDuration(test_time_limit), + absl::StrCat(has_enough_time)); + } + } + ++test_counter_; +} + void Runtime::OnTestIterationEnd() { test_iteration_started_ = false; while (watchdog_spinlock_.test_and_set()) std::this_thread::yield(); diff --git a/fuzztest/internal/runtime.h b/fuzztest/internal/runtime.h index 491fe704..1b83d888 100644 --- a/fuzztest/internal/runtime.h +++ b/fuzztest/internal/runtime.h @@ -176,11 +176,7 @@ class Runtime { UntypedDomain& domain; }; - void SetCurrentTest(const FuzzTest* test, - const Configuration* configuration) { - current_test_ = test; - current_configuration_ = configuration; - } + void SetCurrentTest(const FuzzTest* test, const Configuration* configuration); void OnTestIterationStart(const absl::Time& start_time) { current_iteration_start_time_ = start_time; test_iteration_started_ = true; @@ -243,6 +239,9 @@ class Runtime { RunMode run_mode_ = RunMode::kUnitTest; std::atomic watchdog_thread_started = false; + absl::Time creation_time_ = absl::Now(); + size_t test_counter_ = 0; + bool reporter_enabled_ = false; Args* current_args_ = nullptr; const FuzzTest* current_test_ = nullptr; diff --git a/fuzztest/internal/runtime_test.cc b/fuzztest/internal/runtime_test.cc index 94e54698..0bf2b9a8 100644 --- a/fuzztest/internal/runtime_test.cc +++ b/fuzztest/internal/runtime_test.cc @@ -21,6 +21,7 @@ #include "absl/strings/match.h" #include "absl/time/time.h" #include "./fuzztest/domain_core.h" +#include "./fuzztest/internal/configuration.h" #include "./fuzztest/internal/test_protobuf.pb.h" namespace fuzztest::internal { @@ -37,6 +38,7 @@ TEST(OnFailureTest, Output) { EXPECT_EQ(get_failure(), ""); FuzzTest test({"SUITE_NAME", "TEST_NAME", "FILE", 123}, nullptr); + Configuration configuration; std::tuple args(17, std::string("ABC")); const RuntimeStats stats = {absl::FromUnixNanos(0), 1, 2, 3, 4, 5}; runtime.EnableReporter(&stats, [] { return absl::FromUnixNanos(1979); }); @@ -45,7 +47,7 @@ TEST(OnFailureTest, Output) { GenericDomainCorpusType generic_args( std::in_place_type>, args); Runtime::Args debug_args{generic_args, domain}; - runtime.SetCurrentTest(&test, nullptr); + runtime.SetCurrentTest(&test, &configuration); runtime.SetCurrentArgs(&debug_args); const std::string report = get_failure();