Skip to content

Commit

Permalink
#332 Use UTBotCpp as static analyzer and report results in SARIF (#333)
Browse files Browse the repository at this point in the history
#332 Use UTBotCpp as static analyzer and report results in SARIF

- fix test and file creation logic
- formatting changes
- added "extensionPack" in VSCode plugin
- introduce advertising for MS Sarif plugin if not installed
- formatting and documentation
- fixing progress
  • Loading branch information
alexey-utkin authored Jul 21, 2022
1 parent 6048615 commit 4978445
Show file tree
Hide file tree
Showing 24 changed files with 860 additions and 96 deletions.
2 changes: 1 addition & 1 deletion server/src/KleeGenerator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -346,8 +346,8 @@ void KleeGenerator::parseKTestsToFinalCode(
}
auto predicate =
lineInfo ? lineInfo->predicateInfo : std::optional<LineInfo::PredicateInfo>{};
testsPrinter.genCode(methodDescription, predicate, verbose);

testsPrinter.genCode(methodDescription, predicate, verbose);
}

printer::HeaderPrinter(Paths::getSourceLanguage(tests.sourceFilePath)).print(tests.testHeaderFilePath, tests.sourceFilePath,
Expand Down
47 changes: 41 additions & 6 deletions server/src/KleeRunner.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

#include "Paths.h"
#include "TimeExecStatistics.h"
#include "SARIFGenerator.h"
#include "exceptions/FileNotPresentedInArtifactException.h"
#include "exceptions/FileNotPresentedInCommandsException.h"
#include "tasks/RunKleeTask.h"
Expand Down Expand Up @@ -44,7 +45,9 @@ void KleeRunner::runKlee(const std::vector<tests::TestMethod> &testMethods,
fileToMethods[method.sourceFilePath].push_back(method);
}

std::function<void(tests::Tests &tests)> writeFunctor = [&](tests::Tests &tests) {
nlohmann::json sarifResults;

std::function<void(tests::Tests &tests)> prepareTests = [&](tests::Tests &tests) {
fs::path filePath = tests.sourceFilePath;
const auto &batch = fileToMethods[filePath];
if (!tests.isFilePresentedInCommands) {
Expand Down Expand Up @@ -87,10 +90,22 @@ void KleeRunner::runKlee(const std::vector<tests::TestMethod> &testMethods,
}
generator->parseKTestsToFinalCode(tests, methodNameToReturnTypeMap, ktests, lineInfo,
settingsContext.verbose);

sarif::sarifAddTestsToResults(projectContext, tests, sarifResults);
};

testsWriter->writeTestsWithProgress(testsMap, "Running klee", projectContext.testDirPath,
std::move(writeFunctor));
std::function<void()> prepareTotal = [&]() {
testsWriter->writeReport(sarif::sarifPackResults(sarifResults),
"Sarif Report was created",
projectContext.projectPath / sarif::SARIF_DIR_NAME / sarif::SARIF_FILE_NAME);
};

testsWriter->writeTestsWithProgress(
testsMap,
"Running klee",
projectContext.testDirPath,
std::move(prepareTests),
std::move(prepareTotal));
}

namespace {
Expand Down Expand Up @@ -136,8 +151,13 @@ static void processMethod(MethodKtests &ktestChunk,
LOG_S(WARNING) << "Unable to open .ktestjson file";
continue;
}
UTBotKTest::Status status = Paths::hasError(path) ? UTBotKTest::Status::FAILED
: UTBotKTest::Status::SUCCESS;

const std::vector<fs::path> &errorDescriptorFiles =
Paths::getErrorDescriptors(path);

UTBotKTest::Status status = errorDescriptorFiles.empty()
? UTBotKTest::Status::SUCCESS
: UTBotKTest::Status::FAILED;
std::vector<ConcretizedObject> kTestObjects(
ktestData->objects, ktestData->objects + ktestData->n_objects);

Expand All @@ -146,7 +166,22 @@ static void processMethod(MethodKtests &ktestChunk,
return UTBotKTestObject{ kTestObject };
});

ktestChunk[method].emplace_back(objects, status);
std::vector<std::string> errorDescriptors = CollectionUtils::transform(
errorDescriptorFiles, [](const fs::path &errorFile) {
std::ifstream fileWithError(errorFile.c_str(), std::ios_base::in);
std::string content((std::istreambuf_iterator<char>(fileWithError)),
std::istreambuf_iterator<char>());

const std::string &errorId = errorFile.stem().extension().string();
if (!errorId.empty()) {
// skip leading dot
content += "\n" + sarif::ERROR_ID_KEY + ":" + errorId.substr(1);
}
return content;
});


ktestChunk[method].emplace_back(objects, status, errorDescriptors);
}
}
}
Expand Down
22 changes: 15 additions & 7 deletions server/src/Paths.cpp
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
#include "Paths.h"

#include "ProjectContext.h"
#include "utils/StringUtils.h"
#include "utils/CLIUtils.h"
#include "utils/StringUtils.h"

#include "loguru.h"

Expand Down Expand Up @@ -117,10 +117,12 @@ namespace Paths {

//region klee

static fs::path errorFile(const fs::path &path, std::string const& suffix) {
return replaceExtension(path, StringUtils::stringFormat(".%s.err", suffix));
}

static bool errorFileExists(const fs::path &path, std::string const& suffix) {
fs::path file = replaceExtension(
path, StringUtils::stringFormat(".%s.err", suffix));
return fs::exists(file);
return fs::exists(errorFile(path, suffix));
}

bool hasInternalError(const fs::path &path) {
Expand All @@ -133,7 +135,7 @@ namespace Paths {
[&path](auto const &suffix) { return errorFileExists(path, suffix); });
}

bool hasError(const fs::path &path) {
std::vector<fs::path> getErrorDescriptors(const fs::path &path) {
static const auto internalErrorSuffixes = {
"abort",
"assert",
Expand All @@ -149,8 +151,14 @@ namespace Paths {
"uncaught_exception",
"unexpected_exception"
};
return std::any_of(internalErrorSuffixes.begin(), internalErrorSuffixes.end(),
[&path](auto const &suffix) { return errorFileExists(path, suffix); });

std::vector<fs::path> errFiles;
for (const auto &suffix : internalErrorSuffixes) {
if (errorFileExists(path, suffix)) {
errFiles.emplace_back(errorFile(path, suffix));
}
}
return errFiles;
}

fs::path kleeOutDirForEntrypoints(const utbot::ProjectContext &projectContext, const fs::path &projectTmpPath,
Expand Down
3 changes: 2 additions & 1 deletion server/src/Paths.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

#include "utils/path/FileSystemPath.h"
#include <optional>
#include <vector>
#include <unordered_set>

namespace Paths {
Expand Down Expand Up @@ -236,7 +237,7 @@ namespace Paths {

bool hasInternalError(fs::path const &path);

bool hasError(fs::path const &path);
std::vector<fs::path> getErrorDescriptors(fs::path const &path);

fs::path kleeOutDirForEntrypoints(const utbot::ProjectContext &projectContext, const fs::path &projectTmpPath,
const fs::path &srcFilePath, const std::string &methodName = "");
Expand Down
204 changes: 204 additions & 0 deletions server/src/SARIFGenerator.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,204 @@
#include "SARIFGenerator.h"
#include "Paths.h"

#include "loguru.h"

#include <fstream>
#include <regex>
#include <unordered_map>

using namespace tests;

namespace sarif {
// Here is a temporary solution that restores the correct project-relative path from
// the abstract relative path, provided by KLEE in stack trace inside a `XXXX.err` file.
// There is no clear reason why KLEE is using the wrong base for relative path.
// The correct way to extract the full path for a stack file is in checking entries like
// !820 = !DIFile(filename: "test/suites/cli/complex_structs.c", directory: "/home/utbot/tmp/UTBotCpp/server")
// in upper laying file `assembly.ll`; then we may call the `fs::relative(src, path)`.
// For example the function call:
// getInProjectPath("/home/utbot/tmp/UTBotCpp/server/test/suites/object-file",
// "test/suites/object-file/op/source2.c")
// returns
// "op/source2.c"
fs::path getInProjectPath(const fs::path &path, const fs::path &src) {
fs::path relToProject;
auto p = path.begin();
auto s = src.begin();
bool foundStartFragment = false;
while (p != path.end() && s != src.end()) {
if (*p == *s) {
foundStartFragment = true;
++s;
} else if (foundStartFragment) {
break;
}
++p;
}
if (foundStartFragment && p == path.end()) {
while (s != src.end()) {
relToProject = relToProject / *s;
++s;
}
}
return relToProject;
}

void sarifAddTestsToResults(const utbot::ProjectContext &projectContext,
const Tests &tests,
json &results) {
LOG_S(INFO) << "{stack";
for (const auto &it : tests.methods) {
for (const auto &methodTestCase : it.second.testCases) {
json result;
const std::vector<std::string> &descriptors = methodTestCase.errorDescriptors;
std::string key;
std::string value;
json stackLocations;
json codeFlowsLocations;
json testLocation;
bool canAddThisTestToSARIF = false;
for (const std::string &descriptor : descriptors) {
std::stringstream streamOfDescriptor(descriptor);
std::string lineInDescriptor;
bool firstCallInStack = false;
while (getline(streamOfDescriptor, lineInDescriptor)) {
if (lineInDescriptor.empty() || lineInDescriptor[0] == '#')
continue;
if (isspace(lineInDescriptor[0])) {
if (key == "Stack") {
const std::regex stack_regex(
R"regex(\s+#(.*) in ([^ ]*) [(][^)]*[)] at ([^:]*):(\d+))regex");
std::smatch stack_match;
if (!std::regex_match(lineInDescriptor, stack_match, stack_regex)) {
LOG_S(ERROR) << "wrong `Stack` line: " << lineInDescriptor;
} else {
const fs::path &srcPath = fs::path(stack_match[3]);
const fs::path &relPathInProject = getInProjectPath(projectContext.projectPath, srcPath);
const fs::path &fullPathInProject = projectContext.projectPath / relPathInProject;
if (Paths::isSubPathOf(projectContext.buildDir, fullPathInProject)) {
continue;
}
if (!relPathInProject.empty() && fs::exists(fullPathInProject)) {
// stackLocations from project source
json locationWrapper;
locationWrapper["module"] = "project";
{
json location;
location["physicalLocation"]["artifactLocation"]["uri"] = relPathInProject;
location["physicalLocation"]["artifactLocation"]["uriBaseId"] = "%SRCROOT%";
location["physicalLocation"]["region"]["startLine"] = std::stoi(stack_match[4]); // line number
// commented, duplicated in message
// location["logicalLocations"][0]["fullyQualifiedName"] = stack_match[2]; // call name
location["message"]["text"] = stack_match[2].str() + std::string(" (source)"); // info for ANALYSIS STEP
if (firstCallInStack) {
firstCallInStack = false;
result["locations"].push_back(location);
stackLocations["message"]["text"] = "UTBot generated";
codeFlowsLocations["message"]["text"] = "UTBot generated";
}
locationWrapper["location"] = location;
}
stackLocations["frames"].push_back(locationWrapper);
codeFlowsLocations["locations"].push_back(locationWrapper);
} else if (firstCallInStack) {
// stackLocations from runtime, that is called by tested function
json locationWrapper;
locationWrapper["module"] = "external";
{
json location;
location["physicalLocation"]["artifactLocation"] ["uri"] = srcPath.filename(); // just a name
location["physicalLocation"]["artifactLocation"] ["uriBaseId"] = "%PATH%";
location["physicalLocation"]["region"]["startLine"] = std::stoi(stack_match[4]); // line number
// commented, duplicated in message
// location["logicalLocations"][0]["fullyQualifiedName"] = stack_match[2]; // call name
location["message"]["text"] = stack_match[2].str() + std::string(" (external)"); // info for ANALYSIS STEP
locationWrapper["location"] = location;
}
stackLocations["frames"].push_back(locationWrapper);
codeFlowsLocations["locations"].push_back(locationWrapper);
} else {
// the rest is the KLEE calls that are not applicable for navigation
LOG_S(INFO) << "Skip path in stack frame :" << srcPath;
}
}
}
} else {
size_t pos = lineInDescriptor.find(':');
if (pos == std::string::npos) {
LOG_S(ERROR) << "no key:" << lineInDescriptor;
} else {
if (key == "Stack") {
// Check stack validity
if (firstCallInStack) {
LOG_S(ERROR) << "no visible stack in descriptor:" << descriptor;
} else {
canAddThisTestToSARIF = true;
}
}
firstCallInStack = true;

key = lineInDescriptor.substr(0, pos);
value = lineInDescriptor.substr(pos + 1);
if (key == "Error") {
result["message"]["text"] = value;
result["level"] = "error";
result["kind"] = "fail";
} else if (key == ERROR_ID_KEY) {
result["ruleId"] = value;
} else if (key == "Stack") {
stackLocations = json();
codeFlowsLocations = json();
} else if (key == TEST_FILE_KEY) {
testLocation = json();
testLocation["physicalLocation"]["artifactLocation"]["uri"] = fs::relative(value, projectContext.projectPath);
testLocation["physicalLocation"]["artifactLocation"]["uriBaseId"] = "%SRCROOT%";
} else if (key == TEST_LINE_KEY) {
testLocation["physicalLocation"]["region"]["startLine"] = std::stoi(value); // line number
} else if (key == TEST_NAME_KEY) {
// commented, duplicated in message
// testLocation["logicalLocations"][0]["fullyQualifiedName"] = value; // call name
testLocation["message"]["text"] = value + std::string(" (test)"); // info for ANALYSIS STEP
{
json locationWrapper;
locationWrapper["location"] = testLocation;
locationWrapper["module"] = "test";

stackLocations["frames"].push_back(locationWrapper);
codeFlowsLocations["locations"].push_back(locationWrapper);
}
}
}
}
}
}

if (canAddThisTestToSARIF) {
result["stacks"].push_back(stackLocations);
result["codeFlows"][0]["threadFlows"].push_back(codeFlowsLocations);
results.push_back(result);
}
}
}
LOG_S(INFO) << "}stack";
}

std::string sarifPackResults(const json &results) {
// POINT 3
json sarifJson;
sarifJson["$schema"] = "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json";
sarifJson["version"] = "2.1.0";
{
json runs;
{
json runAkaTestCase;
runAkaTestCase["tool"]["driver"]["name"] = "UTBotCpp";
runAkaTestCase["tool"]["driver"]["informationUri"] = "https://utbot.org";
runAkaTestCase["results"] = results;
runs.push_back(runAkaTestCase);
}
sarifJson["runs"] = runs;
}
return sarifJson.dump(2);
}
}
19 changes: 19 additions & 0 deletions server/src/SARIFGenerator.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#include "Tests.h"
#include "ProjectContext.h"

namespace sarif {
const std::string PREFIX_FOR_JSON_PATH = "// UTBOT_TEST_GENERATOR (function name,test index): ";

const std::string TEST_FILE_KEY = "TestFile";
const std::string TEST_LINE_KEY = "TestLine";
const std::string TEST_NAME_KEY = "TestName";
const std::string ERROR_ID_KEY = "ErrorId";

const std::string SARIF_DIR_NAME = "codeAnalysis";
const std::string SARIF_FILE_NAME = "project_code_analysis.sarif";

void sarifAddTestsToResults(const utbot::ProjectContext &projectContext,
const tests::Tests &tests,
nlohmann::json &results);
std::string sarifPackResults(const nlohmann::json &results);
}
Loading

0 comments on commit 4978445

Please sign in to comment.