From 49784458e9d1e664615d36f1b76eaf021f11bc1f Mon Sep 17 00:00:00 2001 From: Alexey Utkin Date: Thu, 21 Jul 2022 16:11:21 +0300 Subject: [PATCH] #332 Use UTBotCpp as static analyzer and report results in SARIF (#333) #332 Use UTBotCpp as static analyzer and report results in SARIF - fix test and file creation logic - formatting changes - added "extensionPack" in VSCode plugin - introduce advertising for MS Sarif plugin if not installed - formatting and documentation - fixing progress --- server/src/KleeGenerator.cpp | 2 +- server/src/KleeRunner.cpp | 47 +++- server/src/Paths.cpp | 22 +- server/src/Paths.h | 3 +- server/src/SARIFGenerator.cpp | 204 +++++++++++++++ server/src/SARIFGenerator.h | 19 ++ server/src/Tests.cpp | 25 +- server/src/Tests.h | 41 ++- server/src/printers/TestsPrinter.cpp | 113 ++++++-- server/src/printers/TestsPrinter.h | 9 +- server/src/streams/tests/CLITestsWriter.cpp | 24 +- server/src/streams/tests/CLITestsWriter.h | 5 +- .../src/streams/tests/ServerTestsWriter.cpp | 39 ++- server/src/streams/tests/ServerTestsWriter.h | 9 +- server/src/streams/tests/TestsWriter.cpp | 38 +++ server/src/streams/tests/TestsWriter.h | 13 +- server/src/utils/ExecUtils.h | 4 +- server/src/utils/JsonUtils.cpp | 4 + server/src/utils/path/FileSystemPath.h | 6 + server/test/framework/CLI_Tests.cpp | 29 +++ .../codeAnalysis/project_code_analysis.sarif | 244 ++++++++++++++++++ vscode-plugin/package.json | 8 +- .../src/config/notificationMessages.ts | 5 + .../src/responses/responseHandler.ts | 43 ++- 24 files changed, 860 insertions(+), 96 deletions(-) create mode 100644 server/src/SARIFGenerator.cpp create mode 100644 server/src/SARIFGenerator.h create mode 100644 server/test/suites/cli/goldenImage/codeAnalysis/project_code_analysis.sarif diff --git a/server/src/KleeGenerator.cpp b/server/src/KleeGenerator.cpp index 4dbdf407b..1980a822b 100644 --- a/server/src/KleeGenerator.cpp +++ b/server/src/KleeGenerator.cpp @@ -346,8 +346,8 @@ void KleeGenerator::parseKTestsToFinalCode( } auto predicate = lineInfo ? lineInfo->predicateInfo : std::optional{}; - testsPrinter.genCode(methodDescription, predicate, verbose); + testsPrinter.genCode(methodDescription, predicate, verbose); } printer::HeaderPrinter(Paths::getSourceLanguage(tests.sourceFilePath)).print(tests.testHeaderFilePath, tests.sourceFilePath, diff --git a/server/src/KleeRunner.cpp b/server/src/KleeRunner.cpp index 61f7e1b5e..3c3443e75 100644 --- a/server/src/KleeRunner.cpp +++ b/server/src/KleeRunner.cpp @@ -2,6 +2,7 @@ #include "Paths.h" #include "TimeExecStatistics.h" +#include "SARIFGenerator.h" #include "exceptions/FileNotPresentedInArtifactException.h" #include "exceptions/FileNotPresentedInCommandsException.h" #include "tasks/RunKleeTask.h" @@ -44,7 +45,9 @@ void KleeRunner::runKlee(const std::vector &testMethods, fileToMethods[method.sourceFilePath].push_back(method); } - std::function writeFunctor = [&](tests::Tests &tests) { + nlohmann::json sarifResults; + + std::function prepareTests = [&](tests::Tests &tests) { fs::path filePath = tests.sourceFilePath; const auto &batch = fileToMethods[filePath]; if (!tests.isFilePresentedInCommands) { @@ -87,10 +90,22 @@ void KleeRunner::runKlee(const std::vector &testMethods, } generator->parseKTestsToFinalCode(tests, methodNameToReturnTypeMap, ktests, lineInfo, settingsContext.verbose); + + sarif::sarifAddTestsToResults(projectContext, tests, sarifResults); }; - testsWriter->writeTestsWithProgress(testsMap, "Running klee", projectContext.testDirPath, - std::move(writeFunctor)); + std::function prepareTotal = [&]() { + testsWriter->writeReport(sarif::sarifPackResults(sarifResults), + "Sarif Report was created", + projectContext.projectPath / sarif::SARIF_DIR_NAME / sarif::SARIF_FILE_NAME); + }; + + testsWriter->writeTestsWithProgress( + testsMap, + "Running klee", + projectContext.testDirPath, + std::move(prepareTests), + std::move(prepareTotal)); } namespace { @@ -136,8 +151,13 @@ static void processMethod(MethodKtests &ktestChunk, LOG_S(WARNING) << "Unable to open .ktestjson file"; continue; } - UTBotKTest::Status status = Paths::hasError(path) ? UTBotKTest::Status::FAILED - : UTBotKTest::Status::SUCCESS; + + const std::vector &errorDescriptorFiles = + Paths::getErrorDescriptors(path); + + UTBotKTest::Status status = errorDescriptorFiles.empty() + ? UTBotKTest::Status::SUCCESS + : UTBotKTest::Status::FAILED; std::vector kTestObjects( ktestData->objects, ktestData->objects + ktestData->n_objects); @@ -146,7 +166,22 @@ static void processMethod(MethodKtests &ktestChunk, return UTBotKTestObject{ kTestObject }; }); - ktestChunk[method].emplace_back(objects, status); + std::vector errorDescriptors = CollectionUtils::transform( + errorDescriptorFiles, [](const fs::path &errorFile) { + std::ifstream fileWithError(errorFile.c_str(), std::ios_base::in); + std::string content((std::istreambuf_iterator(fileWithError)), + std::istreambuf_iterator()); + + const std::string &errorId = errorFile.stem().extension().string(); + if (!errorId.empty()) { + // skip leading dot + content += "\n" + sarif::ERROR_ID_KEY + ":" + errorId.substr(1); + } + return content; + }); + + + ktestChunk[method].emplace_back(objects, status, errorDescriptors); } } } diff --git a/server/src/Paths.cpp b/server/src/Paths.cpp index a56cd93db..7204b6466 100644 --- a/server/src/Paths.cpp +++ b/server/src/Paths.cpp @@ -1,8 +1,8 @@ #include "Paths.h" #include "ProjectContext.h" -#include "utils/StringUtils.h" #include "utils/CLIUtils.h" +#include "utils/StringUtils.h" #include "loguru.h" @@ -117,10 +117,12 @@ namespace Paths { //region klee + static fs::path errorFile(const fs::path &path, std::string const& suffix) { + return replaceExtension(path, StringUtils::stringFormat(".%s.err", suffix)); + } + static bool errorFileExists(const fs::path &path, std::string const& suffix) { - fs::path file = replaceExtension( - path, StringUtils::stringFormat(".%s.err", suffix)); - return fs::exists(file); + return fs::exists(errorFile(path, suffix)); } bool hasInternalError(const fs::path &path) { @@ -133,7 +135,7 @@ namespace Paths { [&path](auto const &suffix) { return errorFileExists(path, suffix); }); } - bool hasError(const fs::path &path) { + std::vector getErrorDescriptors(const fs::path &path) { static const auto internalErrorSuffixes = { "abort", "assert", @@ -149,8 +151,14 @@ namespace Paths { "uncaught_exception", "unexpected_exception" }; - return std::any_of(internalErrorSuffixes.begin(), internalErrorSuffixes.end(), - [&path](auto const &suffix) { return errorFileExists(path, suffix); }); + + std::vector errFiles; + for (const auto &suffix : internalErrorSuffixes) { + if (errorFileExists(path, suffix)) { + errFiles.emplace_back(errorFile(path, suffix)); + } + } + return errFiles; } fs::path kleeOutDirForEntrypoints(const utbot::ProjectContext &projectContext, const fs::path &projectTmpPath, diff --git a/server/src/Paths.h b/server/src/Paths.h index 2288a8120..14e9b5cbb 100644 --- a/server/src/Paths.h +++ b/server/src/Paths.h @@ -10,6 +10,7 @@ #include "utils/path/FileSystemPath.h" #include +#include #include namespace Paths { @@ -236,7 +237,7 @@ namespace Paths { bool hasInternalError(fs::path const &path); - bool hasError(fs::path const &path); + std::vector getErrorDescriptors(fs::path const &path); fs::path kleeOutDirForEntrypoints(const utbot::ProjectContext &projectContext, const fs::path &projectTmpPath, const fs::path &srcFilePath, const std::string &methodName = ""); diff --git a/server/src/SARIFGenerator.cpp b/server/src/SARIFGenerator.cpp new file mode 100644 index 000000000..27364bdac --- /dev/null +++ b/server/src/SARIFGenerator.cpp @@ -0,0 +1,204 @@ +#include "SARIFGenerator.h" +#include "Paths.h" + +#include "loguru.h" + +#include +#include +#include + +using namespace tests; + +namespace sarif { + // Here is a temporary solution that restores the correct project-relative path from + // the abstract relative path, provided by KLEE in stack trace inside a `XXXX.err` file. + // There is no clear reason why KLEE is using the wrong base for relative path. + // The correct way to extract the full path for a stack file is in checking entries like + // !820 = !DIFile(filename: "test/suites/cli/complex_structs.c", directory: "/home/utbot/tmp/UTBotCpp/server") + // in upper laying file `assembly.ll`; then we may call the `fs::relative(src, path)`. + // For example the function call: + // getInProjectPath("/home/utbot/tmp/UTBotCpp/server/test/suites/object-file", + // "test/suites/object-file/op/source2.c") + // returns + // "op/source2.c" + fs::path getInProjectPath(const fs::path &path, const fs::path &src) { + fs::path relToProject; + auto p = path.begin(); + auto s = src.begin(); + bool foundStartFragment = false; + while (p != path.end() && s != src.end()) { + if (*p == *s) { + foundStartFragment = true; + ++s; + } else if (foundStartFragment) { + break; + } + ++p; + } + if (foundStartFragment && p == path.end()) { + while (s != src.end()) { + relToProject = relToProject / *s; + ++s; + } + } + return relToProject; + } + + void sarifAddTestsToResults(const utbot::ProjectContext &projectContext, + const Tests &tests, + json &results) { + LOG_S(INFO) << "{stack"; + for (const auto &it : tests.methods) { + for (const auto &methodTestCase : it.second.testCases) { + json result; + const std::vector &descriptors = methodTestCase.errorDescriptors; + std::string key; + std::string value; + json stackLocations; + json codeFlowsLocations; + json testLocation; + bool canAddThisTestToSARIF = false; + for (const std::string &descriptor : descriptors) { + std::stringstream streamOfDescriptor(descriptor); + std::string lineInDescriptor; + bool firstCallInStack = false; + while (getline(streamOfDescriptor, lineInDescriptor)) { + if (lineInDescriptor.empty() || lineInDescriptor[0] == '#') + continue; + if (isspace(lineInDescriptor[0])) { + if (key == "Stack") { + const std::regex stack_regex( + R"regex(\s+#(.*) in ([^ ]*) [(][^)]*[)] at ([^:]*):(\d+))regex"); + std::smatch stack_match; + if (!std::regex_match(lineInDescriptor, stack_match, stack_regex)) { + LOG_S(ERROR) << "wrong `Stack` line: " << lineInDescriptor; + } else { + const fs::path &srcPath = fs::path(stack_match[3]); + const fs::path &relPathInProject = getInProjectPath(projectContext.projectPath, srcPath); + const fs::path &fullPathInProject = projectContext.projectPath / relPathInProject; + if (Paths::isSubPathOf(projectContext.buildDir, fullPathInProject)) { + continue; + } + if (!relPathInProject.empty() && fs::exists(fullPathInProject)) { + // stackLocations from project source + json locationWrapper; + locationWrapper["module"] = "project"; + { + json location; + location["physicalLocation"]["artifactLocation"]["uri"] = relPathInProject; + location["physicalLocation"]["artifactLocation"]["uriBaseId"] = "%SRCROOT%"; + location["physicalLocation"]["region"]["startLine"] = std::stoi(stack_match[4]); // line number + // commented, duplicated in message + // location["logicalLocations"][0]["fullyQualifiedName"] = stack_match[2]; // call name + location["message"]["text"] = stack_match[2].str() + std::string(" (source)"); // info for ANALYSIS STEP + if (firstCallInStack) { + firstCallInStack = false; + result["locations"].push_back(location); + stackLocations["message"]["text"] = "UTBot generated"; + codeFlowsLocations["message"]["text"] = "UTBot generated"; + } + locationWrapper["location"] = location; + } + stackLocations["frames"].push_back(locationWrapper); + codeFlowsLocations["locations"].push_back(locationWrapper); + } else if (firstCallInStack) { + // stackLocations from runtime, that is called by tested function + json locationWrapper; + locationWrapper["module"] = "external"; + { + json location; + location["physicalLocation"]["artifactLocation"] ["uri"] = srcPath.filename(); // just a name + location["physicalLocation"]["artifactLocation"] ["uriBaseId"] = "%PATH%"; + location["physicalLocation"]["region"]["startLine"] = std::stoi(stack_match[4]); // line number + // commented, duplicated in message + // location["logicalLocations"][0]["fullyQualifiedName"] = stack_match[2]; // call name + location["message"]["text"] = stack_match[2].str() + std::string(" (external)"); // info for ANALYSIS STEP + locationWrapper["location"] = location; + } + stackLocations["frames"].push_back(locationWrapper); + codeFlowsLocations["locations"].push_back(locationWrapper); + } else { + // the rest is the KLEE calls that are not applicable for navigation + LOG_S(INFO) << "Skip path in stack frame :" << srcPath; + } + } + } + } else { + size_t pos = lineInDescriptor.find(':'); + if (pos == std::string::npos) { + LOG_S(ERROR) << "no key:" << lineInDescriptor; + } else { + if (key == "Stack") { + // Check stack validity + if (firstCallInStack) { + LOG_S(ERROR) << "no visible stack in descriptor:" << descriptor; + } else { + canAddThisTestToSARIF = true; + } + } + firstCallInStack = true; + + key = lineInDescriptor.substr(0, pos); + value = lineInDescriptor.substr(pos + 1); + if (key == "Error") { + result["message"]["text"] = value; + result["level"] = "error"; + result["kind"] = "fail"; + } else if (key == ERROR_ID_KEY) { + result["ruleId"] = value; + } else if (key == "Stack") { + stackLocations = json(); + codeFlowsLocations = json(); + } else if (key == TEST_FILE_KEY) { + testLocation = json(); + testLocation["physicalLocation"]["artifactLocation"]["uri"] = fs::relative(value, projectContext.projectPath); + testLocation["physicalLocation"]["artifactLocation"]["uriBaseId"] = "%SRCROOT%"; + } else if (key == TEST_LINE_KEY) { + testLocation["physicalLocation"]["region"]["startLine"] = std::stoi(value); // line number + } else if (key == TEST_NAME_KEY) { + // commented, duplicated in message + // testLocation["logicalLocations"][0]["fullyQualifiedName"] = value; // call name + testLocation["message"]["text"] = value + std::string(" (test)"); // info for ANALYSIS STEP + { + json locationWrapper; + locationWrapper["location"] = testLocation; + locationWrapper["module"] = "test"; + + stackLocations["frames"].push_back(locationWrapper); + codeFlowsLocations["locations"].push_back(locationWrapper); + } + } + } + } + } + } + + if (canAddThisTestToSARIF) { + result["stacks"].push_back(stackLocations); + result["codeFlows"][0]["threadFlows"].push_back(codeFlowsLocations); + results.push_back(result); + } + } + } + LOG_S(INFO) << "}stack"; + } + + std::string sarifPackResults(const json &results) { + // POINT 3 + json sarifJson; + sarifJson["$schema"] = "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json"; + sarifJson["version"] = "2.1.0"; + { + json runs; + { + json runAkaTestCase; + runAkaTestCase["tool"]["driver"]["name"] = "UTBotCpp"; + runAkaTestCase["tool"]["driver"]["informationUri"] = "https://utbot.org"; + runAkaTestCase["results"] = results; + runs.push_back(runAkaTestCase); + } + sarifJson["runs"] = runs; + } + return sarifJson.dump(2); + } +} diff --git a/server/src/SARIFGenerator.h b/server/src/SARIFGenerator.h new file mode 100644 index 000000000..c02499396 --- /dev/null +++ b/server/src/SARIFGenerator.h @@ -0,0 +1,19 @@ +#include "Tests.h" +#include "ProjectContext.h" + +namespace sarif { + const std::string PREFIX_FOR_JSON_PATH = "// UTBOT_TEST_GENERATOR (function name,test index): "; + + const std::string TEST_FILE_KEY = "TestFile"; + const std::string TEST_LINE_KEY = "TestLine"; + const std::string TEST_NAME_KEY = "TestName"; + const std::string ERROR_ID_KEY = "ErrorId"; + + const std::string SARIF_DIR_NAME = "codeAnalysis"; + const std::string SARIF_FILE_NAME = "project_code_analysis.sarif"; + + void sarifAddTestsToResults(const utbot::ProjectContext &projectContext, + const tests::Tests &tests, + nlohmann::json &results); + std::string sarifPackResults(const nlohmann::json &results); +} diff --git a/server/src/Tests.cpp b/server/src/Tests.cpp index 1b6b8795b..6a0347ac1 100644 --- a/server/src/Tests.cpp +++ b/server/src/Tests.cpp @@ -22,13 +22,12 @@ const Tests::MethodParam &tests::Tests::getStdinMethodParam() { return stdinMethodParam; } -Tests::MethodDescription::MethodDescription() : suiteTestCases{ - {Tests::DEFAULT_SUITE_NAME, std::vector()}, - {Tests::ERROR_SUITE_NAME, std::vector()} -}, codeText{ - {Tests::DEFAULT_SUITE_NAME, std::string()}, - {Tests::ERROR_SUITE_NAME, std::string()} -} {} +Tests::MethodDescription::MethodDescription() + : suiteTestCases{ { Tests::DEFAULT_SUITE_NAME, std::vector() }, + { Tests::ERROR_SUITE_NAME, std::vector() } }, + codeText{ { Tests::DEFAULT_SUITE_NAME, std::string() }, + { Tests::ERROR_SUITE_NAME, std::string() } } { +} static std::string makeDecimalConstant(std::string value, const std::string &typeName) { if (typeName == "long") { @@ -558,7 +557,6 @@ void KTestObjectParser::addToOrder(const std::vector &objects, } std::string message = "Don't find object " + paramName + " in objects array"; LOG_S(WARNING) << message; - return; } void KTestObjectParser::assignTypeUnnamedVar(Tests::MethodTestCase &testCase, @@ -678,12 +676,13 @@ void KTestObjectParser::parseTestCases(const UTBotKTestList &cases, } int caseCounter = 0; + int testIndex = 0; for (const auto &case_ : cases) { try { std::stringstream traceStream; traceStream << "Test case #" << (++caseCounter) << ":\n"; std::string suiteName = getSuiteName(case_.status, lineInfo); - Tests::MethodTestCase testCase{suiteName}; + Tests::MethodTestCase testCase{testIndex, suiteName}; std::vector paramValues; Tests::TestCaseDescription testCaseDescription = parseTestCaseParameters(case_, methodDescription, @@ -695,7 +694,7 @@ void KTestObjectParser::parseTestCases(const UTBotKTestList &cases, methodDescription.params.empty()) { std::swap(testCase.paramValues, testCaseDescription.funcParamValues); } else { - // if all of the data characters are not printable the case is skipped + // if all the data characters are not printable the case is skipped continue; } std::swap(testCase.classPreValues, testCaseDescription.classPreValues); @@ -709,6 +708,9 @@ void KTestObjectParser::parseTestCases(const UTBotKTestList &cases, std::swap(testCase.objects, testCaseDescription.objects); std::swap(testCase.lazyAddressToName, testCaseDescription.lazyAddressToName); std::swap(testCase.lazyReferences, testCaseDescription.lazyReferences); + + testCase.errorDescriptors = case_.errorDescriptors; + if (filterByLineFlag) { auto view = testCaseDescription.kleePathFlagSymbolicValue.view; if (!view || view->getEntryValue(nullptr) != "1") { @@ -740,7 +742,8 @@ void KTestObjectParser::parseTestCases(const UTBotKTestList &cases, assignTypeStubVar(testCase, methodDescription); methodDescription.testCases.push_back(testCase); - methodDescription.suiteTestCases[testCase.suiteName].push_back(testCase); + methodDescription.suiteTestCases[testCase.suiteName].push_back(testCase.testIndex); + ++testIndex; } catch (const UnImplementedException &e) { LOG_S(WARNING) << "Skipping test case: " << e.what(); } catch (const NoSuchTypeException &e) { diff --git a/server/src/Tests.h b/server/src/Tests.h index 1a6d26351..046db7daa 100644 --- a/server/src/Tests.h +++ b/server/src/Tests.h @@ -6,18 +6,21 @@ #include "types/Types.h" #include "utils/CollectionUtils.h" #include "utils/PrinterUtils.h" +#include "json.hpp" #include #include -#include "json.hpp" #include #include #include #include -#include -#include #include +#include +#include +#include + +using json = nlohmann::json; namespace tests { class StructValueView; @@ -60,10 +63,20 @@ namespace tests { }; std::vector objects; Status status; - - UTBotKTest(std::vector objects, Status status) - : objects(std::move(objects)), status(status) { + std::vector errorDescriptors; + + UTBotKTest(const std::vector &objects, + const Status &status, + std::vector &errorDescriptors) : + objects(objects), + status(status), + errorDescriptors(errorDescriptors) + {} + + bool isError() { + return !errorDescriptors.empty(); } + }; using UTBotKTestList = std::vector; @@ -348,10 +361,13 @@ namespace tests { std::vector lazyParams; std::vector lazyValues; TestCaseParamValue() = default; - TestCaseParamValue(std::string name, - std::optional alignment, - std::shared_ptr view) - : name(std::move(name)), alignment(alignment), view(std::move(view)) {}; + + TestCaseParamValue(const std::string &_name, + const std::optional &_alignment, + const std::shared_ptr &_view) + : name(_name), + alignment(_alignment), + view(_view) {} }; struct TestCaseDescription { @@ -378,7 +394,9 @@ namespace tests { }; struct MethodTestCase { + int testIndex; // from 0 std::string suiteName; + std::string testName; // filled by test generator std::vector globalPreValues; std::vector globalPostValues; @@ -398,6 +416,7 @@ namespace tests { TestCaseParamValue returnValue; std::optional classPreValues; std::optional classPostValues; + std::vector errorDescriptors; [[nodiscard]] bool isError() const; }; @@ -428,7 +447,7 @@ namespace tests { typedef std::unordered_map> FPointerMap; FPointerMap functionPointers; std::vector testCases; - typedef std::unordered_map> SuiteNameToTestCasesMap; + typedef std::unordered_map> SuiteNameToTestCasesMap; SuiteNameToTestCasesMap suiteTestCases; bool operator==(const MethodDescription &other) const; diff --git a/server/src/printers/TestsPrinter.cpp b/server/src/printers/TestsPrinter.cpp index a2c5a119e..5ae4321ff 100644 --- a/server/src/printers/TestsPrinter.cpp +++ b/server/src/printers/TestsPrinter.cpp @@ -1,8 +1,9 @@ #include "TestsPrinter.h" #include "Paths.h" -#include "utils/ArgumentsUtils.h" +#include "SARIFGenerator.h" #include "utils/Copyright.h" +#include "utils/JsonUtils.h" #include "visitors/ParametrizedAssertsVisitor.h" #include "visitors/VerboseAssertsParamVisitor.h" #include "visitors/VerboseAssertsReturnValueVisitor.h" @@ -11,6 +12,7 @@ #include "loguru.h" +using json = nlohmann::json; using printer::TestsPrinter; TestsPrinter::TestsPrinter(const types::TypesHandler *typesHandler, utbot::Language srcLanguage) : Printer(srcLanguage) , typesHandler(typesHandler) { @@ -59,7 +61,64 @@ void TestsPrinter::joinToFinalCode(Tests &tests, const fs::path& generatedHeader tests.regressionMethodsNumber = printSuiteAndReturnMethodsCount(Tests::DEFAULT_SUITE_NAME, tests.methods); tests.errorMethodsNumber = printSuiteAndReturnMethodsCount(Tests::ERROR_SUITE_NAME, tests.methods); ss << RB(); - tests.code = ss.str(); + printFinalCodeAndAlterJson(tests); +} + +void TestsPrinter::printFinalCodeAndAlterJson(Tests &tests) { + int line_count = 0; + std::string line; + while (getline(ss, line)) { + if (line.rfind(sarif::PREFIX_FOR_JSON_PATH, 0) != 0) { + // ordinal string + tests.code.append(line); + tests.code.append("\n"); + ++line_count; + } else { + // anchor for SARIF ala testFilePath,lineThatCallsTestedFunction + std::string nameAndTestIndex = + line.substr(sarif::PREFIX_FOR_JSON_PATH.size()); + int pos = nameAndTestIndex.find(','); + if (pos != -1) { + std::string name = nameAndTestIndex.substr(0, pos); + int testIndex = -1; + try { + testIndex = std::stoi(nameAndTestIndex.substr(pos + 1)); + } catch (std::logic_error &e) { + // ignore + } + Tests::MethodsMap::iterator it = tests.methods.find(name); + if (it != tests.methods.end() && testIndex >= 0) { + Tests::MethodDescription &methodDescription = it.value(); + std::vector &testCases = methodDescription.testCases; + if (testIndex < testCases.size()) { + Tests::MethodTestCase &testCase = testCases[testIndex]; + auto &descriptors = testCase.errorDescriptors; + if (testCase.errorDescriptors.empty()) { + LOG_S(ERROR) << "no error info for test case: " + << name + << ", test #" + << testIndex; + continue; + } + std::stringstream ssFromTestCallInfo; + ssFromTestCallInfo + << sarif::TEST_FILE_KEY << ":" << tests.testSourceFilePath.c_str() << std::endl + << sarif::TEST_LINE_KEY << ":" << line_count << std::endl + << sarif::TEST_NAME_KEY << ":" << testCase.suiteName + << "." + << testCase.testName + << std::endl; + + descriptors.emplace_back(ssFromTestCallInfo.str()); + // ok + continue; + } + } + } + LOG_S(ERROR) << "wrong SARIF anchor (need {testFilePath,lineThatCallsTestedFunction}): " + << line; + } + } } std::uint32_t TestsPrinter::printSuiteAndReturnMethodsCount(const std::string &suiteName, const Tests::MethodsMap &methods) { @@ -112,18 +171,31 @@ void TestsPrinter::genCode(Tests::MethodDescription &methodDescription, resetStream(); } +static std::string getTestName(const Tests::MethodDescription &methodDescription, int testNum) { + std::string renamedMethodDescription = KleeUtils::getRenamedOperator(methodDescription.name); + std::string testBaseName = methodDescription.isClassMethod() + ? StringUtils::stringFormat("%s_%s", + methodDescription.classObj->type.typeName(), + renamedMethodDescription) + : renamedMethodDescription; + + return printer::Printer::concat(testBaseName, "_test_", testNum); +} + void TestsPrinter::genCodeBySuiteName(const std::string &targetSuiteName, Tests::MethodDescription &methodDescription, const std::optional& predicateInfo, bool verbose, int &testNum) { - const auto& testCases = methodDescription.suiteTestCases[targetSuiteName]; + auto &testCases = methodDescription.suiteTestCases[targetSuiteName]; if (testCases.empty()) { return; } - for (auto &testCase : testCases) { - testNum++; - testHeader(testCase.suiteName, methodDescription, testNum); + for (int testCaseIndex : testCases) { + ++testNum; + Tests::MethodTestCase &testCase = methodDescription.testCases[testCaseIndex]; + testCase.testName = getTestName(methodDescription, testNum); + testHeader(testCase); redirectStdin(methodDescription, testCase, verbose); if (verbose) { genVerboseTestCase(methodDescription, testCase, predicateInfo); @@ -152,6 +224,8 @@ void TestsPrinter::genVerboseTestCase(const Tests::MethodDescription &methodDesc ss << NL; } TestsPrinter::verboseFunctionCall(methodDescription, testCase); + markTestedFunctionCallIfNeed(methodDescription.name, testCase); + ss << NL; if (testCase.isError()) { ss << TAB_N() @@ -247,21 +321,8 @@ void TestsPrinter::genHeaders(Tests &tests, const fs::path& generatedHeaderPath) writeAccessPrivateMacros(typesHandler, tests, true); } -static std::string getTestName(const Tests::MethodDescription &methodDescription, int testNum) { - std::string renamedMethodDescription = KleeUtils::getRenamedOperator(methodDescription.name); - std::string testBaseName = methodDescription.isClassMethod() - ? StringUtils::stringFormat("%s_%s", methodDescription.classObj->type.typeName(), - renamedMethodDescription) - : renamedMethodDescription; - - return printer::Printer::concat(testBaseName, "_test_", testNum); -} - -void TestsPrinter::testHeader(const std::string &scopeName, - const Tests::MethodDescription &methodDescription, - int testNum) { - std::string testName = getTestName(methodDescription, testNum); - strFunctionCall("TEST", { scopeName, testName }, NL) << LB(false); +void TestsPrinter::testHeader(const Tests::MethodTestCase &testCase) { + strFunctionCall("TEST", { testCase.suiteName, testCase.testName }, NL) << LB(false); } void TestsPrinter::redirectStdin(const Tests::MethodDescription &methodDescription, @@ -540,6 +601,7 @@ void TestsPrinter::parametrizedAsserts(const Tests::MethodDescription &methodDes const std::optional& predicateInfo) { auto visitor = visitor::ParametrizedAssertsVisitor(typesHandler, this, predicateInfo, testCase.isError()); visitor.visit(methodDescription, testCase); + markTestedFunctionCallIfNeed(methodDescription.name, testCase); if (!testCase.isError()) { globalParamsAsserts(methodDescription, testCase); classAsserts(methodDescription, testCase); @@ -547,6 +609,15 @@ void TestsPrinter::parametrizedAsserts(const Tests::MethodDescription &methodDes } } +void TestsPrinter::markTestedFunctionCallIfNeed(const std::string &name, + const Tests::MethodTestCase &testCase) { + if (testCase.errorDescriptors.empty()) { + // cannot generate stack for error + return; + } + ss << sarif::PREFIX_FOR_JSON_PATH << name << "," << testCase.testIndex << NL; +} + std::vector TestsPrinter::methodParametersListParametrized(const Tests::MethodDescription &methodDescription, const Tests::MethodTestCase &testCase) { std::vector args; diff --git a/server/src/printers/TestsPrinter.h b/server/src/printers/TestsPrinter.h index 067cac469..6ac5a30fe 100644 --- a/server/src/printers/TestsPrinter.h +++ b/server/src/printers/TestsPrinter.h @@ -42,9 +42,7 @@ namespace printer { const Tests::MethodTestCase &testCase, const std::optional &predicateInfo); - void testHeader(const std::string &scopeName, - const tests::Tests::MethodDescription &methodDescription, - int testNum); + void testHeader(const Tests::MethodTestCase &testCase); void redirectStdin(const tests::Tests::MethodDescription &methodDescription, const Tests::MethodTestCase &testCase, @@ -91,6 +89,11 @@ namespace printer { const Tests::MethodTestCase &testCase, const std::optional& predicateInfo); + void markTestedFunctionCallIfNeed(const std::string &name, + const Tests::MethodTestCase &testCase); + + void printFinalCodeAndAlterJson(Tests &tests); + std::vector methodParametersListParametrized(const tests::Tests::MethodDescription &methodDescription, const Tests::MethodTestCase &testCase); diff --git a/server/src/streams/tests/CLITestsWriter.cpp b/server/src/streams/tests/CLITestsWriter.cpp index 45a8aa79a..a7ff301e2 100644 --- a/server/src/streams/tests/CLITestsWriter.cpp +++ b/server/src/streams/tests/CLITestsWriter.cpp @@ -8,28 +8,22 @@ void CLITestsWriter::writeTestsWithProgress(tests::TestsMap &testMap, const std::string &message, const fs::path &testDirPath, - std::function &&functor) { - size_t size = testMap.size(); + std::function &&prepareTests, + std::function &&prepareTotal) { std::cout << message << std::endl; int totalTestsCounter = 0; - for (auto it = testMap.begin(); it != testMap.end(); it++) { + for (auto it = testMap.begin(); it != testMap.end(); ++it) { tests::Tests& tests = it.value(); - functor(tests); + prepareTests(tests); if (writeTestFile(tests, testDirPath)) { - totalTestsCounter += 1; - auto generatedMessage = StringUtils::stringFormat("%s test file generated", tests.testFilename); - LOG_S(INFO) << generatedMessage; + ++totalTestsCounter; + LOG_S(INFO) << tests.testFilename << " test file generated"; } } - std::string finalMessage; - if (totalTestsCounter == 1) { - finalMessage = StringUtils::stringFormat("%d test file generated.", totalTestsCounter); - } else { - finalMessage = StringUtils::stringFormat("%d test files generated.", totalTestsCounter); - } - LOG_S(INFO) << finalMessage; - + prepareTotal(); + LOG_S(INFO) << "total test files generated: " << totalTestsCounter; } + bool CLITestsWriter::writeTestFile(const tests::Tests &tests, const fs::path &testDirPath) { fs::path testFilePath = testDirPath / tests.relativeFileDir / tests.testFilename; FileSystemUtils::writeToFile(testFilePath, tests.code); diff --git a/server/src/streams/tests/CLITestsWriter.h b/server/src/streams/tests/CLITestsWriter.h index cee50b57b..e784c693b 100644 --- a/server/src/streams/tests/CLITestsWriter.h +++ b/server/src/streams/tests/CLITestsWriter.h @@ -11,9 +11,10 @@ class CLITestsWriter : public TestsWriter { explicit CLITestsWriter(): TestsWriter(nullptr) {}; void writeTestsWithProgress(tests::TestsMap &testMap, - std::string const &message, + const std::string &message, const fs::path &testDirPath, - std::function &&functor) override; + std::function &&prepareTests, + std::function &&prepareTotal) override; private: static bool writeTestFile(const tests::Tests &tests, const fs::path &testDirPath); diff --git a/server/src/streams/tests/ServerTestsWriter.cpp b/server/src/streams/tests/ServerTestsWriter.cpp index 52f91f7c4..2c71a3613 100644 --- a/server/src/streams/tests/ServerTestsWriter.cpp +++ b/server/src/streams/tests/ServerTestsWriter.cpp @@ -4,22 +4,26 @@ #include "loguru.h" +#include +#include + void ServerTestsWriter::writeTestsWithProgress(tests::TestsMap &testMap, - std::string const &message, + const std::string &message, const fs::path &testDirPath, - std::function &&functor) { - + std::function &&prepareTests, + std::function &&prepareTotal) { size_t size = testMap.size(); writeProgress(message); int totalTestsCounter = 0; - for (auto it = testMap.begin(); it != testMap.end(); it++) { + for (auto it = testMap.begin(); it != testMap.end(); ++it) { tests::Tests &tests = it.value(); ExecUtils::throwIfCancelled(); - functor(tests); - if (writeFileAndSendResponse(tests, testDirPath, message, 100.0 / size, false)) { - totalTestsCounter += 1; + prepareTests(tests); + if (writeFileAndSendResponse(tests, testDirPath, message, (100.0 * totalTestsCounter) / size, false)) { + ++totalTestsCounter; } } + prepareTotal(); writeCompleted(testMap, totalTestsCounter); } @@ -60,3 +64,24 @@ bool ServerTestsWriter::writeFileAndSendResponse(const tests::Tests &tests, writeMessage(response); return isAnyTestsGenerated; } + +void ServerTestsWriter::writeReport(const std::string &content, + const std::string &message, + const fs::path &pathToStore) const +{ + if (synchronizeCode || fs::exists(pathToStore)) { + testsgen::TestsResponse response; + if (synchronizeCode) { + TestsWriter::writeReport(content, message, pathToStore); + } + auto testSource = response.add_testsources(); + testSource->set_filepath(pathToStore); + if (synchronizeCode) { + testSource->set_code(content); + } + LOG_S(INFO) << message; + auto progress = GrpcUtils::createProgress(message, 100, false); + response.set_allocated_progress(progress.release()); + writeMessage(response); + } +} diff --git a/server/src/streams/tests/ServerTestsWriter.h b/server/src/streams/tests/ServerTestsWriter.h index 8b825b6a6..b6504bf29 100644 --- a/server/src/streams/tests/ServerTestsWriter.h +++ b/server/src/streams/tests/ServerTestsWriter.h @@ -14,9 +14,14 @@ class ServerTestsWriter : public TestsWriter { : TestsWriter(writer), synchronizeCode(synchronizeCode) {}; void writeTestsWithProgress(tests::TestsMap &testMap, - std::string const &message, + const std::string &message, const fs::path &testDirPath, - std::function &&functor) override; + std::function &&prepareTests, + std::function &&prepareTotal) override; + + void writeReport(const std::string &content, + const std::string &message, + const fs::path &pathToStore) const override; private: [[nodiscard]] virtual bool writeFileAndSendResponse(const tests::Tests &tests, diff --git a/server/src/streams/tests/TestsWriter.cpp b/server/src/streams/tests/TestsWriter.cpp index d166a1ea1..900e4e70d 100644 --- a/server/src/streams/tests/TestsWriter.cpp +++ b/server/src/streams/tests/TestsWriter.cpp @@ -1,5 +1,6 @@ #include "TestsWriter.h" +#include "SARIFGenerator.h" #include "utils/FileSystemUtils.h" #include "loguru.h" @@ -16,3 +17,40 @@ void TestsWriter::writeCompleted(const tests::TestsMap &testMap, int totalTestsC } writeProgress(finalMessage, 100.0, true); } + +void TestsWriter::writeReport(const std::string &content, + const std::string &message, + const fs::path &pathToStore) const +{ + try { + backupIfExists(pathToStore); + } catch (const std::exception &e) { + LOG_S(ERROR) << e.what() + << ": problem in `writeReport` with " + << pathToStore; + } + FileSystemUtils::writeToFile(pathToStore, content); +} + +template +std::time_t to_time_t(TP tp) +{ + using namespace std::chrono; + auto sctp = time_point_cast(tp - TP::clock::now() + system_clock::now()); + return system_clock::to_time_t(sctp); +} + +void TestsWriter::backupIfExists(const fs::path &filePath) { + if (fs::exists(filePath)) { + std::filesystem::file_time_type ftime = fs::last_write_time(filePath); + time_t tt = to_time_t(ftime); + tm *gmt = localtime(&tt); + + std::stringstream nfn; + nfn << filePath.stem().c_str() << "-" << std::put_time(gmt, "%Y%m%d%H%M%S") + << filePath.extension().c_str(); + + LOG_S(INFO) << "Backup previous report to " << nfn.str(); + fs::rename(filePath, filePath.parent_path() / nfn.str()); + } +} diff --git a/server/src/streams/tests/TestsWriter.h b/server/src/streams/tests/TestsWriter.h index 970b931fb..2e080f7f8 100644 --- a/server/src/streams/tests/TestsWriter.h +++ b/server/src/streams/tests/TestsWriter.h @@ -1,7 +1,6 @@ #ifndef UNITTESTBOT_TESTSWRITER_H #define UNITTESTBOT_TESTSWRITER_H - #include "Tests.h" #include "streams/BaseWriter.h" #include "streams/IStreamWriter.h" @@ -16,14 +15,20 @@ class TestsWriter : public utbot::ServerWriter { explicit TestsWriter(grpc::ServerWriter *writer); virtual void writeTestsWithProgress(tests::TestsMap &testMap, - std::string const &message, + const std::string &message, const fs::path &testDirPath, - std::function &&functor) = 0; + std::function &&prepareTests, + std::function &&prepareTotal) = 0; + + virtual void writeReport(const std::string &content, + const std::string &message, + const fs::path &pathToStore) const; + + static void backupIfExists(const fs::path &filePath); protected: void writeCompleted(tests::TestsMap const &testMap, int totalTestsCounter); -private: }; diff --git a/server/src/utils/ExecUtils.h b/server/src/utils/ExecUtils.h index 3f9f2ed3c..fbe1801ec 100644 --- a/server/src/utils/ExecUtils.h +++ b/server/src/utils/ExecUtils.h @@ -42,10 +42,12 @@ namespace ExecUtils { Functor &&functor) { size_t size = iterable.size(); progressWriter->writeProgress(message); + size_t step = 0; for (auto &&it : iterable) { throwIfCancelled(); functor(it); - progressWriter->writeProgress(message, 100.0 / size); + progressWriter->writeProgress(message, (100.0 * step) / size); + ++step; } } diff --git a/server/src/utils/JsonUtils.cpp b/server/src/utils/JsonUtils.cpp index 8e85bf305..8ace4364a 100644 --- a/server/src/utils/JsonUtils.cpp +++ b/server/src/utils/JsonUtils.cpp @@ -6,6 +6,7 @@ #include "utils/path/FileSystemPath.h" #include +#include namespace JsonUtils { nlohmann::json getJsonFromFile(const fs::path &path) { @@ -15,6 +16,9 @@ namespace JsonUtils { try { nlohmann::json coverageJson = nlohmann::json::parse(buffer.str()); return coverageJson; + } catch (const std::exception &e) { + LOG_S(ERROR) << e.what() << ": " << buffer.str() << " in: " << path.string(); + throw e; } catch (...) { LOG_S(ERROR) << buffer.str(); throw; diff --git a/server/src/utils/path/FileSystemPath.h b/server/src/utils/path/FileSystemPath.h index 3924caddb..d67fea6eb 100644 --- a/server/src/utils/path/FileSystemPath.h +++ b/server/src/utils/path/FileSystemPath.h @@ -149,6 +149,7 @@ namespace fs { friend bool create_directories( const path& p ); friend void copy( const path& from, const path& to); friend void copy( const path& from, const path& to, std::filesystem::copy_options options ); + friend void rename( const path& from, const path& to); friend bool copy_file( const path& from, const path& to, copy_options options ); @@ -221,6 +222,11 @@ namespace fs { copy(from.path_, to.path_, options); } + inline void rename( const path& from, + const path& to) { + rename(from.path_, to.path_); + } + inline std::filesystem::file_time_type last_write_time(const path& p) { return last_write_time(p.path_); } diff --git a/server/test/framework/CLI_Tests.cpp b/server/test/framework/CLI_Tests.cpp index 97cc89ea2..2325493ab 100644 --- a/server/test/framework/CLI_Tests.cpp +++ b/server/test/framework/CLI_Tests.cpp @@ -2,6 +2,7 @@ #include "BaseTest.h" #include "KleeGenerator.h" +#include "SARIFGenerator.h" #include "Paths.h" #include "Server.h" #include "TestUtils.h" @@ -100,10 +101,38 @@ namespace { } }; + std::size_t replaceAll(std::string& inout, std::string_view what, std::string_view with) + { + std::size_t count{}; + for (std::string::size_type pos{}; + inout.npos != (pos = inout.find(what.data(), pos, what.length())); + pos += with.length(), ++count) { + inout.replace(pos, what.length(), with.data(), with.length()); + } + return count; + } + + std::string getNormalizedContent(const fs::path &name) { + EXPECT_TRUE(fs::exists(name)) << "File " << name.c_str() << " don't exists!"; + std::ifstream src(name, std::ios_base::in); + std::string content((std::istreambuf_iterator(src)), std::istreambuf_iterator()); + replaceAll(content, "\r\n", "\n"); + return content; + } + + void compareFiles(const fs::path &golden, const fs::path &real) { + ASSERT_EQ(getNormalizedContent(golden), getNormalizedContent(real)); + } + TEST_F(CLI_Test, Generate_Project_Tests) { + fs::remove(suitePath / sarif::SARIF_DIR_NAME / sarif::SARIF_FILE_NAME); + runCommandLine({ "./utbot", "generate", "--project-path", suitePath, "--build-dir", buildDirectoryName, "project" }); checkTestDirectory(allProjectTestFiles); + + compareFiles( suitePath / "goldenImage" / sarif::SARIF_DIR_NAME / sarif::SARIF_FILE_NAME, + suitePath / sarif::SARIF_DIR_NAME / sarif::SARIF_FILE_NAME); } TEST_F(CLI_Test, Generate_File_Tests) { diff --git a/server/test/suites/cli/goldenImage/codeAnalysis/project_code_analysis.sarif b/server/test/suites/cli/goldenImage/codeAnalysis/project_code_analysis.sarif new file mode 100644 index 000000000..bbb95cb26 --- /dev/null +++ b/server/test/suites/cli/goldenImage/codeAnalysis/project_code_analysis.sarif @@ -0,0 +1,244 @@ +{ + "$schema": "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json", + "runs": [ + { + "results": [ + { + "codeFlows": [ + { + "threadFlows": [ + { + "locations": [ + { + "location": { + "message": { + "text": "buggy_function2 (source)" + }, + "physicalLocation": { + "artifactLocation": { + "uri": "assertion_failures.c", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 5 + } + } + }, + "module": "project" + }, + { + "location": { + "message": { + "text": "error.buggy_function2_test_1 (test)" + }, + "physicalLocation": { + "artifactLocation": { + "uri": "tests/assertion_failures_dot_c_test.cpp", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 15 + } + } + }, + "module": "test" + } + ], + "message": { + "text": "UTBot generated" + } + } + ] + } + ], + "kind": "fail", + "level": "error", + "locations": [ + { + "message": { + "text": "buggy_function2 (source)" + }, + "physicalLocation": { + "artifactLocation": { + "uri": "assertion_failures.c", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 5 + } + } + } + ], + "message": { + "text": " ASSERTION FAIL: a < 7" + }, + "ruleId": "assert", + "stacks": [ + { + "frames": [ + { + "location": { + "message": { + "text": "buggy_function2 (source)" + }, + "physicalLocation": { + "artifactLocation": { + "uri": "assertion_failures.c", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 5 + } + } + }, + "module": "project" + }, + { + "location": { + "message": { + "text": "error.buggy_function2_test_1 (test)" + }, + "physicalLocation": { + "artifactLocation": { + "uri": "tests/assertion_failures_dot_c_test.cpp", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 15 + } + } + }, + "module": "test" + } + ], + "message": { + "text": "UTBot generated" + } + } + ] + }, + { + "codeFlows": [ + { + "threadFlows": [ + { + "locations": [ + { + "location": { + "message": { + "text": "buggy_function2 (source)" + }, + "physicalLocation": { + "artifactLocation": { + "uri": "assertion_failures.c", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 4 + } + } + }, + "module": "project" + }, + { + "location": { + "message": { + "text": "error.buggy_function2_test_2 (test)" + }, + "physicalLocation": { + "artifactLocation": { + "uri": "tests/assertion_failures_dot_c_test.cpp", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 20 + } + } + }, + "module": "test" + } + ], + "message": { + "text": "UTBot generated" + } + } + ] + } + ], + "kind": "fail", + "level": "error", + "locations": [ + { + "message": { + "text": "buggy_function2 (source)" + }, + "physicalLocation": { + "artifactLocation": { + "uri": "assertion_failures.c", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 4 + } + } + } + ], + "message": { + "text": " ASSERTION FAIL: a == 7" + }, + "ruleId": "assert", + "stacks": [ + { + "frames": [ + { + "location": { + "message": { + "text": "buggy_function2 (source)" + }, + "physicalLocation": { + "artifactLocation": { + "uri": "assertion_failures.c", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 4 + } + } + }, + "module": "project" + }, + { + "location": { + "message": { + "text": "error.buggy_function2_test_2 (test)" + }, + "physicalLocation": { + "artifactLocation": { + "uri": "tests/assertion_failures_dot_c_test.cpp", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 20 + } + } + }, + "module": "test" + } + ], + "message": { + "text": "UTBot generated" + } + } + ] + } + ], + "tool": { + "driver": { + "informationUri": "https://utbot.org", + "name": "UTBotCpp" + } + } + } + ], + "version": "2.1.0" +} \ No newline at end of file diff --git a/vscode-plugin/package.json b/vscode-plugin/package.json index 167ed4b54..776f56b60 100644 --- a/vscode-plugin/package.json +++ b/vscode-plugin/package.json @@ -1,5 +1,5 @@ { - "name": "UTBotCpp", + "name": "utbotcpp", "displayName": "UnitTestBot for C/C++", "publisher": "UnitTestBot", "repository": "https://github.com/UnitTestBot/UTBotCpp", @@ -580,5 +580,9 @@ "randomstring": "1.2.2", "source-map-support": "0.5.21", "typescript": "3.9.4" - } + }, + "extensionPack": [ + "Natizyskunk.sftp", + "MS-SarifVSCode.sarif-viewer" + ] } diff --git a/vscode-plugin/src/config/notificationMessages.ts b/vscode-plugin/src/config/notificationMessages.ts index ab7b72947..7b012d404 100644 --- a/vscode-plugin/src/config/notificationMessages.ts +++ b/vscode-plugin/src/config/notificationMessages.ts @@ -11,6 +11,11 @@ export const serverIsDeadError = "UTBot server doesn't respond. Check the connec export const grpcConnectionLostError = "No connection established"; export const targetNotUsed = "There is no used target. Use any in UTBot Targets window, please."; +// {SARIF +export const defaultSARIFViewer = "MS-SarifVSCode.sarif-viewer"; +export const intstallSARIFViewer = "Please, install MS Sarif Viewer from https://marketplace.visualstudio.com/items?itemName=" + defaultSARIFViewer; +// }SARIF + export function showErrorMessage(err: any): void { let errorMessage = getErrorMessage(err); diff --git a/vscode-plugin/src/responses/responseHandler.ts b/vscode-plugin/src/responses/responseHandler.ts index 604fae35e..3acc49d9f 100644 --- a/vscode-plugin/src/responses/responseHandler.ts +++ b/vscode-plugin/src/responses/responseHandler.ts @@ -3,8 +3,12 @@ import { Prefs } from "../config/prefs"; import { CoverageAndResultsResponse, ProjectConfigResponse, StubsResponse, TestsResponse } from "../proto-ts/testgen_pb"; import * as pathUtils from '../utils/pathUtils'; import * as vs from 'vscode'; +import * as path from 'path'; +import * as fs from 'fs'; import { ExtensionLogger } from "../logger"; import { TestsRunner } from "../runner/testsRunner"; +import {Uri} from "vscode"; +import * as messages from "../config/notificationMessages"; const { logger } = ExtensionLogger; @@ -29,6 +33,7 @@ export class TestsResponseHandler implements ResponseHandler { public async handle(response: TestsResponse): Promise { const testsSourceList = response.getTestsourcesList(); + testsSourceList.forEach(testsSourceInfo => { this.testsRunner.testResultsVizualizer.clearTestsByTestFileName(testsSourceInfo.getFilepath(), false); }); @@ -43,8 +48,7 @@ export class TestsResponseHandler implements ResponseHandler { await vs.workspace.fs.writeFile(stubfile, Buffer.from(stub.getCode())); })); } - const testsFiles = testsSourceList; - await Promise.all((testsFiles).map(async (test) => { + await Promise.all((testsSourceList).map(async (test) => { const localPath = pathUtils.substituteLocalPath(test.getFilepath()); const testfile = vs.Uri.file(localPath); @@ -55,7 +59,42 @@ export class TestsResponseHandler implements ResponseHandler { } else { logger.info(`Generated test file ${localPath}`); } + + const isSarifReport = testfile.path.endsWith("project_code_analysis.sarif"); + if (isSarifReport && fs.existsSync(testfile.fsPath)) { + const ctime = fs.lstatSync(testfile.fsPath).ctime; + + // eslint-disable-next-line no-inner-declarations + function pad2(num: number): string { + return ("0" + num).slice(-2); + } + + const newName = "project_code_analysis-" + + ctime.getFullYear() + + pad2(ctime.getMonth() + 1) + + pad2(ctime.getDate()) + + pad2(ctime.getHours()) + + pad2(ctime.getMinutes()) + + pad2(ctime.getSeconds()) + + ".sarif"; + await vs.workspace.fs.rename(testfile, Uri.file(path.join(path.dirname(testfile.fsPath), newName))); + } + await vs.workspace.fs.writeFile(testfile, Buffer.from(test.getCode())); + if (isSarifReport) { + const sarifExt = vs.extensions.getExtension(messages.defaultSARIFViewer); + // eslint-disable-next-line eqeqeq + if (sarifExt == null) { + messages.showWarningMessage(messages.intstallSARIFViewer); + } else { + if (!sarifExt.isActive) { + await sarifExt.activate(); + } + await sarifExt.exports.openLogs([ + testfile, + ]); + } + } return testfile; })); }