From 6d195f16312e1890f48b0fea62344d516f3af152 Mon Sep 17 00:00:00 2001 From: gamesh411 Date: Wed, 31 Jul 2019 08:39:35 +0200 Subject: [PATCH] Add CTU On-Demand analysis support Add an option to enable on-demand parsing of needed ASTs during CTU analysis, and another option to specify the compilation database used. The option CTUOnDemandParsing is a boolean flag, which enables the new AST-loading mode. Option CTUOnDemandParsingDatabase is a string, which should be the path of the compilation database used for on-demand parsing. The compilation database is needed for on-demand mode, because it has all the necessary information to generate the ASTs. Please enter the commit message for your changes. Lines starting --- include/clang/CrossTU/CrossTranslationUnit.h | 32 ++- .../StaticAnalyzer/Core/AnalyzerOptions.def | 15 ++ lib/CrossTU/CMakeLists.txt | 1 + lib/CrossTU/CrossTranslationUnit.cpp | 226 +++++++++++++++--- lib/Frontend/CompilerInvocation.cpp | 6 + lib/StaticAnalyzer/Core/CallEvent.cpp | 11 +- ...> ctu-other.c.externalDefMap.ast-dump.txt} | 0 .../ctu-other.c.externalDefMap.on-the-fly.txt | 6 + ...ctu-other.cpp.externalDefMap.ast-dump.txt} | 0 ...tu-other.cpp.externalDefMap.on-the-fly.txt | 17 ++ test/Analysis/analyzer-config.c | 4 +- test/Analysis/ctu-different-triples.cpp | 2 +- test/Analysis/ctu-main.c | 4 +- test/Analysis/ctu-main.cpp | 2 +- test/Analysis/ctu-on-demand-parsing.c | 75 ++++++ test/Analysis/ctu-on-demand-parsing.cpp | 100 ++++++++ .../Analysis/ctu-unknown-parts-in-triples.cpp | 2 +- .../CrossTU/CrossTranslationUnitTest.cpp | 8 +- 18 files changed, 455 insertions(+), 56 deletions(-) rename test/Analysis/Inputs/{ctu-other.c.externalDefMap.txt => ctu-other.c.externalDefMap.ast-dump.txt} (100%) create mode 100644 test/Analysis/Inputs/ctu-other.c.externalDefMap.on-the-fly.txt rename test/Analysis/Inputs/{ctu-other.cpp.externalDefMap.txt => ctu-other.cpp.externalDefMap.ast-dump.txt} (100%) create mode 100644 test/Analysis/Inputs/ctu-other.cpp.externalDefMap.on-the-fly.txt create mode 100644 test/Analysis/ctu-on-demand-parsing.c create mode 100644 test/Analysis/ctu-on-demand-parsing.cpp diff --git a/include/clang/CrossTU/CrossTranslationUnit.h b/include/clang/CrossTU/CrossTranslationUnit.h index eb1508031e76..57a7b75126bd 100644 --- a/include/clang/CrossTU/CrossTranslationUnit.h +++ b/include/clang/CrossTU/CrossTranslationUnit.h @@ -32,6 +32,10 @@ class FunctionDecl; class NamedDecl; class TranslationUnitDecl; +namespace tooling { +class JSONCompilationDatabase; +} + namespace cross_tu { enum class index_error_code { @@ -41,12 +45,14 @@ enum class index_error_code { multiple_definitions, missing_definition, failed_import, + failed_to_load_compilation_database, failed_to_get_external_ast, failed_to_generate_usr, triple_mismatch, lang_mismatch, lang_dialect_mismatch, - load_threshold_reached + load_threshold_reached, + ambiguous_compile_commands_database }; class IndexError : public llvm::ErrorInfo { @@ -85,7 +91,8 @@ class IndexError : public llvm::ErrorInfo { /// \return Returns a map where the USR is the key and the filepath is the value /// or an error. llvm::Expected> -parseCrossTUIndex(StringRef IndexPath, StringRef CrossTUDir); +parseCrossTUIndex(StringRef IndexPath, StringRef CrossTUDir, + llvm::Optional OnDemandParsingDatabase); std::string createCrossTUIndexString(const llvm::StringMap &Index); @@ -125,7 +132,8 @@ class CrossTranslationUnitContext { llvm::Expected getCrossTUDefinition(const FunctionDecl *FD, StringRef CrossTUDir, StringRef IndexName, bool DisplayCTUProgress, - unsigned CTULoadThreshold); + unsigned CTULoadThreshold, + llvm::Optional OnDemandParsingDatabase); /// This function loads a function definition from an external AST /// file. @@ -141,11 +149,11 @@ class CrossTranslationUnitContext { /// The returned pointer is never a nullptr. /// /// Note that the AST files should also be in the \p CrossTUDir. - llvm::Expected loadExternalAST(StringRef LookupName, - StringRef CrossTUDir, - StringRef IndexName, - bool DisplayCTUProgress, - unsigned CTULoadThreshold); + llvm::Expected + loadExternalAST(StringRef LookupName, StringRef CrossTUDir, + StringRef IndexName, bool DisplayCTUProgress, + unsigned CTULoadThreshold, + llvm::Optional OnDemandParsingDatabase); /// This function merges a definition from a separate AST Unit into /// the current one which was created by the compiler instance that @@ -172,6 +180,11 @@ class CrossTranslationUnitContext { GetImportedFromSourceLocation(const clang::SourceLocation &ToLoc) const; private: + llvm::Expected> + loadASTFromDump(StringRef ASTSourcePath) const; + llvm::Expected> + loadASTOnDemand(StringRef ASTSourcePath) const; + llvm::Error lazyInitCompileCommands(StringRef CompileCommandsFile); void lazyInitImporterSharedSt(TranslationUnitDecl *ToTU); ASTImporter &getOrCreateASTImporter(ASTUnit *Unit); const FunctionDecl *findFunctionInDeclContext(const DeclContext *DC, @@ -190,6 +203,9 @@ class CrossTranslationUnitContext { ASTContext &Context; std::shared_ptr ImporterSharedSt; unsigned NumASTLoaded{0u}; + /// In case of on-demand parsing, the compilation database is parsed and + /// stored. + std::unique_ptr CompileCommands; }; } // namespace cross_tu diff --git a/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def b/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def index 3320282934e5..a48bf9fac00a 100644 --- a/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def +++ b/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def @@ -355,6 +355,21 @@ ANALYZER_OPTION(StringRef, CTUIndexName, "ctu-index-name", "the name of the file containing the CTU index of definitions.", "externalDefMap.txt") +ANALYZER_OPTION(bool, CTUOnDemandParsing, "ctu-on-demand-parsing", + "Whether to parse function definitions from external TUs in " + "an on-demand manner during analysis. When using on-demand " + "parsing there is no need for pre-dumping ASTs. External " + "definition mapping is still needed, and a valid compilation " + "database with compile commands for the external TUs is also " + "necessary. Disabled by default.", + false) + +ANALYZER_OPTION(StringRef, CTUOnDemandParsingDatabase, + "ctu-on-demand-parsing-database", + "The path to the compilation database used for on-demand " + "parsing of ASTs during CTU analysis.", + "compile_commands.json") + ANALYZER_OPTION( StringRef, ModelPath, "model-path", "The analyzer can inline an alternative implementation written in C at the " diff --git a/lib/CrossTU/CMakeLists.txt b/lib/CrossTU/CMakeLists.txt index 632b5072ad6a..f4bf1ae5925c 100644 --- a/lib/CrossTU/CMakeLists.txt +++ b/lib/CrossTU/CMakeLists.txt @@ -10,4 +10,5 @@ add_clang_library(clangCrossTU clangBasic clangFrontend clangIndex + clangTooling ) diff --git a/lib/CrossTU/CrossTranslationUnit.cpp b/lib/CrossTU/CrossTranslationUnit.cpp index 1974ecf0fac3..675a5d49dd05 100644 --- a/lib/CrossTU/CrossTranslationUnit.cpp +++ b/lib/CrossTU/CrossTranslationUnit.cpp @@ -19,12 +19,15 @@ #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/TextDiagnosticPrinter.h" #include "clang/Index/USRGeneration.h" -#include "llvm/ADT/Triple.h" +#include "clang/Tooling/JSONCompilationDatabase.h" +#include "clang/Tooling/Tooling.h" #include "llvm/ADT/Statistic.h" +#include "llvm/ADT/Triple.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/Path.h" #include "llvm/Support/raw_ostream.h" +#include #include #include @@ -101,6 +104,8 @@ class IndexErrorCategory : public std::error_category { return "Failed to import the definition."; case index_error_code::failed_to_get_external_ast: return "Failed to load external AST source."; + case index_error_code::failed_to_load_compilation_database: + return "Failed to load compilation database."; case index_error_code::failed_to_generate_usr: return "Failed to generate USR."; case index_error_code::triple_mismatch: @@ -111,6 +116,9 @@ class IndexErrorCategory : public std::error_category { return "Language dialect mismatch"; case index_error_code::load_threshold_reached: return "Load threshold reached"; + case index_error_code::ambiguous_compile_commands_database: + return "Compile commands database contains multiple references to the " + "same sorce file."; } llvm_unreachable("Unrecognized index_error_code."); } @@ -130,7 +138,8 @@ std::error_code IndexError::convertToErrorCode() const { } llvm::Expected> -parseCrossTUIndex(StringRef IndexPath, StringRef CrossTUDir) { +parseCrossTUIndex(StringRef IndexPath, StringRef CrossTUDir, + llvm::Optional OnDemandParsingDatabase) { std::ifstream ExternalMapFile(IndexPath); if (!ExternalMapFile) return llvm::make_error(index_error_code::missing_index_file, @@ -148,9 +157,14 @@ parseCrossTUIndex(StringRef IndexPath, StringRef CrossTUDir) { return llvm::make_error( index_error_code::multiple_definitions, IndexPath.str(), LineNo); StringRef FileName = LineRef.substr(Pos + 1); - SmallString<256> FilePath = CrossTUDir; - llvm::sys::path::append(FilePath, FileName); - Result[LookupName] = FilePath.str().str(); + // AST-dump based analysis requires a prefixed path. + if (!OnDemandParsingDatabase) { + SmallString<256> FilePath = CrossTUDir; + llvm::sys::path::append(FilePath, FileName); + Result[LookupName] = FilePath.str().str(); + } else { + Result[LookupName] = FileName.str(); + } } else return llvm::make_error( index_error_code::invalid_index_format, IndexPath.str(), LineNo); @@ -203,11 +217,10 @@ CrossTranslationUnitContext::findFunctionInDeclContext(const DeclContext *DC, } llvm::Expected -CrossTranslationUnitContext::getCrossTUDefinition(const FunctionDecl *FD, - StringRef CrossTUDir, - StringRef IndexName, - bool DisplayCTUProgress, - unsigned CTULoadThreshold) { +CrossTranslationUnitContext::getCrossTUDefinition( + const FunctionDecl *FD, StringRef CrossTUDir, StringRef IndexName, + bool DisplayCTUProgress, unsigned CTULoadThreshold, + llvm::Optional OnDemandParsingDatabase) { assert(FD && "FD is missing, bad call to this function!"); assert(!FD->hasBody() && "FD has a definition in current translation unit!"); ++NumGetCTUCalled; @@ -217,7 +230,7 @@ CrossTranslationUnitContext::getCrossTUDefinition(const FunctionDecl *FD, index_error_code::failed_to_generate_usr); llvm::Expected ASTUnitOrError = loadExternalAST(LookupFnName, CrossTUDir, IndexName, DisplayCTUProgress, - CTULoadThreshold); + CTULoadThreshold, OnDemandParsingDatabase); if (!ASTUnitOrError) return ASTUnitOrError.takeError(); ASTUnit *Unit = *ASTUnitOrError; @@ -302,9 +315,136 @@ void CrossTranslationUnitContext::emitCrossTUDiagnostics(const IndexError &IE) { } } +/// Load the AST from a source-file, which is supposed to be located inside the +/// compilation database \p OnDemandParsingCommands. The compilation database +/// can contain the path of the file under the key "file" as an absolute path, +/// or as a relative path. When emitting diagnostics, plist files may contain +/// references to a location in a TU, that is different from the main TU. In +/// such cases, the file path emitted by the DiagnosticEngine is based on how +/// the exact invocation is assembled inside the ClangTool, which performs the +/// building of the ASTs. In order to ensure absolute paths inside the +/// diagnostics, we use the ArgumentsAdjuster API of ClangTool to make sure that +/// the invocation inside ClangTool is always made with an absolute path. \p +/// ASTSourcePath is assumed to be the lookup-name of the file, which comes from +/// the Index. The Index is built by the \p clang-extdef-mapping tool, which is +/// supposed to generate absolute paths. +/// +/// We must have absolute paths inside the plist, because otherwise we would +/// not be able to parse the bug, because we could not find the files with +/// relative paths. The directory of one entry in the compilation db may be +/// different from the directory where the plist is interpreted. +/// +/// Note that as the ClangTool is instantiated with a lookup-vector, which +/// contains a single entry; the supposedly absolute path of the source file. +/// So, the ArgumentAdjuster will only be used on the single corresponding +/// invocation. This garantees that even if two files match in name, but +/// differ in location, only the correct one's invocation will be handled. This +/// is due to the fact that the lookup is done correctly inside the +/// OnDemandParsingDatabase, so it works for already absolute paths given under +/// the "file" entry of the compilation database, but also if a relative path is +/// given. In such a case, the lookup uses the "directory" entry as well to +/// identify the correct file. +llvm::Expected> +CrossTranslationUnitContext::loadASTOnDemand(StringRef ASTSourcePath) const { + + using namespace tooling; + + SmallVector Files; + Files.push_back(ASTSourcePath); + ClangTool Tool(*CompileCommands, Files, CI.getPCHContainerOperations()); + + /// Lambda filter designed to find the source file argument inside an + /// invocation used to build the ASTs, and replace it with its absolute path + /// equivalent. + auto SourcePathNormalizer = [ASTSourcePath](const CommandLineArguments &Args, + StringRef FileName) { + /// Match the argument to the absolute path by checking whether it is a + /// postfix. + auto IsPostfixOfLookup = [ASTSourcePath](const std::string &Arg) { + return ASTSourcePath.rfind(Arg) != llvm::StringRef::npos; + }; + + /// Commandline arguments are modified, and the API dictates the return of + /// a new instance, so copy the original. + CommandLineArguments Result{Args}; + + /// Search for the source file argument. Start from the end as a heuristic, + /// as most invocations tend to contain the source file argument in their + /// latter half. Only the first match is replaced. + auto SourceFilePath = + std::find_if(Result.rbegin(), Result.rend(), IsPostfixOfLookup); + + /// If source file argument could not been found, return the original + /// CommandlineArgumentsInstance. + if (SourceFilePath == Result.rend()) + return Result; + + /// Overwrite the argument with the \p ASTSourcePath, as it is assumed to + /// be the absolute path of the file. + *SourceFilePath = ASTSourcePath.str(); + + return Result; + }; + + Tool.appendArgumentsAdjuster(std::move(SourcePathNormalizer)); + + std::vector> ASTs; + Tool.buildASTs(ASTs); + + /// There is an assumption that the compilation database does not contain + /// multiple entries for the same source file. + if (ASTs.size() > 1) + return llvm::make_error( + index_error_code::ambiguous_compile_commands_database); + + /// Ideally there is exactly one entry in the compilation database that + /// matchse the source file. + if (ASTs.size() != 1) + return llvm::make_error( + index_error_code::failed_to_get_external_ast); + + ASTs[0]->enableSourceFileDiagnostics(); + return std::move(ASTs[0]); +} + +llvm::Expected> +CrossTranslationUnitContext::loadASTFromDump(StringRef ASTFileName) const { + // If no \p OnDemandParsingDatabase is given, try to load from AST dump + // file, as on-demand parsing is disabled. + IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); + TextDiagnosticPrinter *DiagClient = + new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts); + IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); + IntrusiveRefCntPtr Diags( + new DiagnosticsEngine(DiagID, &*DiagOpts, DiagClient)); + + std::unique_ptr LoadedUnit(ASTUnit::LoadFromASTFile( + ASTFileName, CI.getPCHContainerOperations()->getRawReader(), + ASTUnit::LoadEverything, Diags, CI.getFileSystemOpts())); + + if (!LoadedUnit) + return llvm::make_error( + index_error_code::failed_to_get_external_ast); + + return std::move(LoadedUnit); +} + +llvm::Error CrossTranslationUnitContext::lazyInitCompileCommands( + StringRef CompileCommandsFile) { + // Lazily initialize the compilation database. + std::string LoadError; + CompileCommands = tooling::JSONCompilationDatabase::loadFromFile( + CompileCommandsFile, LoadError, + tooling::JSONCommandLineSyntax::AutoDetect); + return CompileCommands ? llvm::Error::success() + : llvm::make_error( + index_error_code::failed_to_get_external_ast); +} + llvm::Expected CrossTranslationUnitContext::loadExternalAST( StringRef LookupName, StringRef CrossTUDir, StringRef IndexName, - bool DisplayCTUProgress, unsigned CTULoadThreshold) { + bool DisplayCTUProgress, unsigned CTULoadThreshold, + llvm::Optional OnDemandParsingDatabase) { // FIXME: The current implementation only supports loading functions with // a lookup name from a single translation unit. If multiple // translation units contains functions with the same lookup name an @@ -320,17 +460,20 @@ llvm::Expected CrossTranslationUnitContext::loadExternalAST( auto FnUnitCacheEntry = FunctionASTUnitMap.find(LookupName); if (FnUnitCacheEntry == FunctionASTUnitMap.end()) { if (FunctionFileMap.empty()) { + SmallString<256> IndexFile = CrossTUDir; if (llvm::sys::path::is_absolute(IndexName)) IndexFile = IndexName; else llvm::sys::path::append(IndexFile, IndexName); + llvm::Expected> IndexOrErr = - parseCrossTUIndex(IndexFile, CrossTUDir); - if (IndexOrErr) - FunctionFileMap = *IndexOrErr; - else + parseCrossTUIndex(IndexFile, CrossTUDir, OnDemandParsingDatabase); + + if (!IndexOrErr) return IndexOrErr.takeError(); + + FunctionFileMap = *IndexOrErr; } auto It = FunctionFileMap.find(LookupName); @@ -338,36 +481,47 @@ llvm::Expected CrossTranslationUnitContext::loadExternalAST( ++NumNotInOtherTU; return llvm::make_error(index_error_code::missing_definition); } - StringRef ASTFileName = It->second; - auto ASTCacheEntry = FileASTUnitMap.find(ASTFileName); + StringRef ASTSource = It->second; + auto ASTCacheEntry = FileASTUnitMap.find(ASTSource); if (ASTCacheEntry == FileASTUnitMap.end()) { - IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); - TextDiagnosticPrinter *DiagClient = - new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts); - IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); - IntrusiveRefCntPtr Diags( - new DiagnosticsEngine(DiagID, &*DiagOpts, DiagClient)); - - std::unique_ptr LoadedUnit(ASTUnit::LoadFromASTFile( - ASTFileName, CI.getPCHContainerOperations()->getRawReader(), - ASTUnit::LoadEverything, Diags, CI.getFileSystemOpts())); - Unit = LoadedUnit.get(); - FileASTUnitMap[ASTFileName] = std::move(LoadedUnit); + // Cache miss. + + if (OnDemandParsingDatabase) { + llvm::Error InitError = + lazyInitCompileCommands(*OnDemandParsingDatabase); + if (InitError) + return std::move(InitError); + } + + llvm::Expected> LoadedUnit = + OnDemandParsingDatabase ? loadASTOnDemand(ASTSource) + : loadASTFromDump(ASTSource); + + if (!LoadedUnit) + return LoadedUnit.takeError(); + + Unit = LoadedUnit->get(); + + // Cache the resulting ASTUnit. + FileASTUnitMap[ASTSource] = std::move(*LoadedUnit); + ++NumASTLoaded; if (DisplayCTUProgress) { - llvm::errs() << "CTU loaded AST file: " - << ASTFileName << "\n"; + llvm::errs() << "CTU loaded AST file: " << ASTSource << "\n"; } } else { Unit = ASTCacheEntry->second.get(); } + // Fill the cache for the lookup name as well. + assert(Unit); FunctionASTUnitMap[LookupName] = Unit; } else { Unit = FnUnitCacheEntry->second; } - if (!Unit) - return llvm::make_error( - index_error_code::failed_to_get_external_ast); + + // Only non-null pointers are cached, because the load operations should only + // finish without an error, if the pointer they load is not null. + assert(Unit); return Unit; } @@ -397,7 +551,7 @@ CrossTranslationUnitContext::importDefinition(const FunctionDecl *FD, }); return llvm::make_error(index_error_code::failed_import); } - auto *ToDecl = cast(*ToDeclOrError); + auto *ToDecl = cast(*ToDeclOrError); assert(ToDecl->hasBody() && "Imported function should have body."); ++NumGetCTUSuccess; diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index 3e6528c25982..6cc080ed309a 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -460,6 +460,12 @@ static void parseAnalyzerConfigs(AnalyzerOptions &AnOpts, Diags->Report(diag::err_analyzer_config_invalid_input) << "ctu-dir" << "a filename"; + if (AnOpts.CTUOnDemandParsing && + !llvm::sys::fs::exists(AnOpts.CTUOnDemandParsingDatabase)) + Diags->Report(diag::err_analyzer_config_invalid_input) + << "ctu-on-demand-parsing-database" + << "a filename"; + if (!AnOpts.ModelPath.empty() && !llvm::sys::fs::is_directory(AnOpts.ModelPath)) Diags->Report(diag::err_analyzer_config_invalid_input) << "model-path" diff --git a/lib/StaticAnalyzer/Core/CallEvent.cpp b/lib/StaticAnalyzer/Core/CallEvent.cpp index d535b0783f13..269eaee0151e 100644 --- a/lib/StaticAnalyzer/Core/CallEvent.cpp +++ b/lib/StaticAnalyzer/Core/CallEvent.cpp @@ -559,10 +559,15 @@ RuntimeDefinition AnyFunctionCall::getRuntimeDefinition() const { cross_tu::CrossTranslationUnitContext &CTUCtx = *Engine.getCrossTranslationUnitContext(); + + Optional OnDemandParsingDatabase; + if (Opts.CTUOnDemandParsing) + OnDemandParsingDatabase = Opts.CTUOnDemandParsingDatabase; + llvm::Expected CTUDeclOrError = - CTUCtx.getCrossTUDefinition(FD, Opts.CTUDir, Opts.CTUIndexName, - Opts.DisplayCTUProgress, - Opts.CTUImportThreshold); + CTUCtx.getCrossTUDefinition( + FD, Opts.CTUDir, Opts.CTUIndexName, Opts.DisplayCTUProgress, + Opts.CTUImportThreshold, OnDemandParsingDatabase); if (!CTUDeclOrError) { handleAllErrors(CTUDeclOrError.takeError(), diff --git a/test/Analysis/Inputs/ctu-other.c.externalDefMap.txt b/test/Analysis/Inputs/ctu-other.c.externalDefMap.ast-dump.txt similarity index 100% rename from test/Analysis/Inputs/ctu-other.c.externalDefMap.txt rename to test/Analysis/Inputs/ctu-other.c.externalDefMap.ast-dump.txt diff --git a/test/Analysis/Inputs/ctu-other.c.externalDefMap.on-the-fly.txt b/test/Analysis/Inputs/ctu-other.c.externalDefMap.on-the-fly.txt new file mode 100644 index 000000000000..0da8d2d09e73 --- /dev/null +++ b/test/Analysis/Inputs/ctu-other.c.externalDefMap.on-the-fly.txt @@ -0,0 +1,6 @@ +c:@F@inlineAsm ctu-other.c +c:@F@g ctu-other.c +c:@F@f ctu-other.c +c:@F@enumCheck ctu-other.c +c:@F@identImplicit ctu-other.c +c:@F@structInProto ctu-other.c diff --git a/test/Analysis/Inputs/ctu-other.cpp.externalDefMap.txt b/test/Analysis/Inputs/ctu-other.cpp.externalDefMap.ast-dump.txt similarity index 100% rename from test/Analysis/Inputs/ctu-other.cpp.externalDefMap.txt rename to test/Analysis/Inputs/ctu-other.cpp.externalDefMap.ast-dump.txt diff --git a/test/Analysis/Inputs/ctu-other.cpp.externalDefMap.on-the-fly.txt b/test/Analysis/Inputs/ctu-other.cpp.externalDefMap.on-the-fly.txt new file mode 100644 index 000000000000..1e9cf5a16cf8 --- /dev/null +++ b/test/Analysis/Inputs/ctu-other.cpp.externalDefMap.on-the-fly.txt @@ -0,0 +1,17 @@ +c:@N@chns@F@chf1#I# ctu-other.cpp +c:@N@myns@N@embed_ns@F@fens#I# ctu-other.cpp +c:@F@g#I# ctu-other.cpp +c:@S@mycls@F@fscl#I#S ctu-other.cpp +c:@S@mycls@F@fcl#I# ctu-other.cpp +c:@S@mycls@F@fvcl#I# ctu-other.cpp +c:@N@myns@S@embed_cls@F@fecl#I# ctu-other.cpp +c:@S@mycls@S@embed_cls2@F@fecl2#I# ctu-other.cpp +c:@S@derived@F@fvcl#I# ctu-other.cpp +c:@F@f#I# ctu-other.cpp +c:@N@myns@F@fns#I# ctu-other.cpp +c:@F@h#I# ctu-other.cpp +c:@F@h_chain#I# ctu-chain.cpp +c:@N@chns@S@chcls@F@chf4#I# ctu-chain.cpp +c:@N@chns@F@chf2#I# ctu-chain.cpp +c:@F@fun_using_anon_struct#I# ctu-other.cpp +c:@F@other_macro_diag#I# ctu-other.cpp diff --git a/test/Analysis/analyzer-config.c b/test/Analysis/analyzer-config.c index 27c19979a70d..4e206eaad49e 100644 --- a/test/Analysis/analyzer-config.c +++ b/test/Analysis/analyzer-config.c @@ -23,6 +23,8 @@ // CHECK-NEXT: ctu-dir = "" // CHECK-NEXT: ctu-import-threshold = 100 // CHECK-NEXT: ctu-index-name = externalDefMap.txt +// CHECK-NEXT: ctu-on-demand-parsing = false +// CHECK-NEXT: ctu-on-demand-parsing-database = compile_commands.json // CHECK-NEXT: display-ctu-progress = false // CHECK-NEXT: eagerly-assume = true // CHECK-NEXT: elide-constructors = true @@ -54,4 +56,4 @@ // CHECK-NEXT: unroll-loops = false // CHECK-NEXT: widen-loops = false // CHECK-NEXT: [stats] -// CHECK-NEXT: num-entries = 51 +// CHECK-NEXT: num-entries = 53 diff --git a/test/Analysis/ctu-different-triples.cpp b/test/Analysis/ctu-different-triples.cpp index dbfa82fb483d..13ae7762d6d7 100644 --- a/test/Analysis/ctu-different-triples.cpp +++ b/test/Analysis/ctu-different-triples.cpp @@ -2,7 +2,7 @@ // RUN: mkdir -p %t/ctudir // RUN: %clang_cc1 -triple x86_64-pc-linux-gnu \ // RUN: -emit-pch -o %t/ctudir/ctu-other.cpp.ast %S/Inputs/ctu-other.cpp -// RUN: cp %S/Inputs/ctu-other.cpp.externalDefMap.txt %t/ctudir/externalDefMap.txt +// RUN: cp %S/Inputs/ctu-other.cpp.externalDefMap.ast-dump.txt %t/ctudir/externalDefMap.txt // RUN: %clang_analyze_cc1 -triple powerpc64-montavista-linux-gnu \ // RUN: -analyzer-checker=core,debug.ExprInspection \ // RUN: -analyzer-config experimental-enable-naive-ctu-analysis=true \ diff --git a/test/Analysis/ctu-main.c b/test/Analysis/ctu-main.c index 94f4e5f24038..5466aae964b2 100644 --- a/test/Analysis/ctu-main.c +++ b/test/Analysis/ctu-main.c @@ -2,7 +2,7 @@ // RUN: mkdir -p %t/ctudir2 // RUN: %clang_cc1 -triple x86_64-pc-linux-gnu \ // RUN: -emit-pch -o %t/ctudir2/ctu-other.c.ast %S/Inputs/ctu-other.c -// RUN: cp %S/Inputs/ctu-other.c.externalDefMap.txt %t/ctudir2/externalDefMap.txt +// RUN: cp %S/Inputs/ctu-other.c.externalDefMap.ast-dump.txt %t/ctudir2/externalDefMap.txt // RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fsyntax-only -std=c89 -analyze \ // RUN: -analyzer-checker=core,debug.ExprInspection \ // RUN: -analyzer-config experimental-enable-naive-ctu-analysis=true \ @@ -56,7 +56,7 @@ void testImplicit() { // Call something with uninitialized from the same function in which the implicit was called. // This is necessary to reproduce a special bug in NoStoreFuncVisitor. int uninitialized; - h(uninitialized); // expected-warning@ctu-main.c:59 {{1st function call argument is an uninitialized value}} + h(uninitialized); // expected-warning{{1st function call argument is an uninitialized value}} } // Tests the import of functions that have a struct parameter diff --git a/test/Analysis/ctu-main.cpp b/test/Analysis/ctu-main.cpp index 2b5e1ab04797..e72ebec4d214 100644 --- a/test/Analysis/ctu-main.cpp +++ b/test/Analysis/ctu-main.cpp @@ -4,7 +4,7 @@ // RUN: -emit-pch -o %t/ctudir/ctu-other.cpp.ast %S/Inputs/ctu-other.cpp // RUN: %clang_cc1 -triple x86_64-pc-linux-gnu \ // RUN: -emit-pch -o %t/ctudir/ctu-chain.cpp.ast %S/Inputs/ctu-chain.cpp -// RUN: cp %S/Inputs/ctu-other.cpp.externalDefMap.txt %t/ctudir/externalDefMap.txt +// RUN: cp %S/Inputs/ctu-other.cpp.externalDefMap.ast-dump.txt %t/ctudir/externalDefMap.txt // RUN: %clang_analyze_cc1 -triple x86_64-pc-linux-gnu \ // RUN: -analyzer-checker=core,debug.ExprInspection \ // RUN: -analyzer-config experimental-enable-naive-ctu-analysis=true \ diff --git a/test/Analysis/ctu-on-demand-parsing.c b/test/Analysis/ctu-on-demand-parsing.c new file mode 100644 index 000000000000..592db59799b1 --- /dev/null +++ b/test/Analysis/ctu-on-demand-parsing.c @@ -0,0 +1,75 @@ +// RUN: rm -rf %t && mkdir %t +// RUN: mkdir -p %t/ctudir2 +// RUN: echo '[{"directory":"%S/Inputs","command":"clang -c ctu-other.c","file":"ctu-other.c"}]' | sed -e 's/\\/\\\\/g' > %t/ctudir2/compile_commands.json +// RUN: %clang_extdef_map %S/Inputs/ctu-other.c > %t/ctudir2/externalDefMap.txt +// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fsyntax-only -std=c89 -analyze \ +// RUN: -analyzer-checker=core,debug.ExprInspection \ +// RUN: -analyzer-config experimental-enable-naive-ctu-analysis=true \ +// RUN: -analyzer-config ctu-dir=%t/ctudir2 \ +// RUN: -analyzer-config ctu-on-demand-parsing=true \ +// RUN: -analyzer-config ctu-on-demand-parsing-database="%t/ctudir2/compile_commands.json" \ +// RUN: -verify %s + +void clang_analyzer_eval(int); + +// Test typedef and global variable in function. +typedef struct { + int a; + int b; +} FooBar; +extern FooBar fb; +int f(int); +void testGlobalVariable() { + clang_analyzer_eval(f(5) == 1); // expected-warning{{TRUE}} +} + +// Test enums. +int enumCheck(void); +enum A { x, + y, + z }; +void testEnum() { + clang_analyzer_eval(x == 0); // expected-warning{{TRUE}} + clang_analyzer_eval(enumCheck() == 42); // expected-warning{{TRUE}} +} + +// Test that asm import does not fail. +int inlineAsm(); +int testInlineAsm() { + return inlineAsm(); +} + +// Test reporting error in a macro. +struct S; +int g(struct S *); +void testMacro(void) { + g(0); // expected-warning@Inputs/ctu-other.c:29 {{Access to field 'a' results in a dereference of a null pointer (loaded from variable 'ctx')}} +} + +void h(int); + +// The external function prototype is incomplete. +// warning:implicit functions are prohibited by c99 +void testImplicit() { + int res = identImplicit(6); + clang_analyzer_eval(res == 6); // expected-warning{{TRUE}} + + // Call something with uninitialized from the same function in which the implicit was called. + // This is necessary to reproduce a special bug in NoStoreFuncVisitor. + int uninitialized; + h(uninitialized); // expected-warning{{1st function call argument is an uninitialized value}} +} + +// Tests the import of functions that have a struct parameter +// defined in its prototype. +struct DataType { + int a; + int b; +}; +int structInProto(struct DataType *d); +void testStructDefInArgument() { + struct DataType d; + d.a = 1; + d.b = 0; + clang_analyzer_eval(structInProto(&d) == 0); // expected-warning{{TRUE}} expected-warning{{FALSE}} +} diff --git a/test/Analysis/ctu-on-demand-parsing.cpp b/test/Analysis/ctu-on-demand-parsing.cpp new file mode 100644 index 000000000000..1243aa6a07ef --- /dev/null +++ b/test/Analysis/ctu-on-demand-parsing.cpp @@ -0,0 +1,100 @@ +// RUN: rm -rf %t && mkdir %t +// RUN: mkdir -p %t/ctudir +// RUN: cp %S/Inputs/ctu-chain.cpp %t/ctudir/ctu-chain.cpp +// RUN: echo '[{"directory":"%S/Inputs","command":"clang++ -c ctu-chain.cpp","file":"ctu-chain.cpp"},{"directory":"%S/Inputs","command":"clang++ -c ctu-other.cpp","file":"ctu-other.cpp"}]' | sed -e 's/\\/\\\\/g' > %t/ctudir/compile_commands.json +// RUN: %clang_extdef_map %S/Inputs/ctu-chain.cpp %S/Inputs/ctu-other.cpp > %t/ctudir/externalDefMap.txt +// RUN: %clang_analyze_cc1 -triple x86_64-pc-linux-gnu \ +// RUN: -analyzer-checker=core,debug.ExprInspection \ +// RUN: -analyzer-config experimental-enable-naive-ctu-analysis=true \ +// RUN: -analyzer-config ctu-dir="%t/ctudir" \ +// RUN: -analyzer-config ctu-on-demand-parsing=true \ +// RUN: -analyzer-config ctu-on-demand-parsing-database="%t/ctudir/compile_commands.json" \ +// RUN: -verify %s +// RUN: %clang_analyze_cc1 -triple x86_64-pc-linux-gnu \ +// RUN: -analyzer-checker=core,debug.ExprInspection \ +// RUN: -analyzer-config experimental-enable-naive-ctu-analysis=true \ +// RUN: -analyzer-config ctu-dir="%t/ctudir" \ +// RUN: -analyzer-config ctu-on-demand-parsing=true \ +// RUN: -analyzer-config ctu-on-demand-parsing-database="%t/ctudir/compile_commands.json" \ +// RUN: -analyzer-config display-ctu-progress=true 2>&1 %s | FileCheck %s + +// CHECK: CTU loaded AST file: {{.*}}ctu-other.cpp +// CHECK: CTU loaded AST file: {{.*}}ctu-chain.cpp + +#include "ctu-hdr.h" + +void clang_analyzer_eval(int); + +int f(int); +int g(int); +int h(int); + +int callback_to_main(int x) { return x + 1; } + +namespace myns { +int fns(int x); + +namespace embed_ns { +int fens(int x); +} + +class embed_cls { +public: + int fecl(int x); +}; +} // namespace myns + +class mycls { +public: + int fcl(int x); + virtual int fvcl(int x); + static int fscl(int x); + + class embed_cls2 { + public: + int fecl2(int x); + }; +}; + +class derived : public mycls { +public: + virtual int fvcl(int x) override; +}; + +namespace chns { +int chf1(int x); +} + +int fun_using_anon_struct(int); +int other_macro_diag(int); + +void test_virtual_functions(mycls *obj) { + // The dynamic type is known. + clang_analyzer_eval(mycls().fvcl(1) == 8); // expected-warning{{TRUE}} + clang_analyzer_eval(derived().fvcl(1) == 9); // expected-warning{{TRUE}} + // We cannot decide about the dynamic type. + clang_analyzer_eval(obj->fvcl(1) == 8); // expected-warning{{FALSE}} expected-warning{{TRUE}} + clang_analyzer_eval(obj->fvcl(1) == 9); // expected-warning{{FALSE}} expected-warning{{TRUE}} +} + +int main() { + clang_analyzer_eval(f(3) == 2); // expected-warning{{TRUE}} + clang_analyzer_eval(f(4) == 3); // expected-warning{{TRUE}} + clang_analyzer_eval(f(5) == 3); // expected-warning{{FALSE}} + clang_analyzer_eval(g(4) == 6); // expected-warning{{TRUE}} + clang_analyzer_eval(h(2) == 8); // expected-warning{{TRUE}} + + clang_analyzer_eval(myns::fns(2) == 9); // expected-warning{{TRUE}} + clang_analyzer_eval(myns::embed_ns::fens(2) == -1); // expected-warning{{TRUE}} + clang_analyzer_eval(mycls().fcl(1) == 6); // expected-warning{{TRUE}} + clang_analyzer_eval(mycls::fscl(1) == 7); // expected-warning{{TRUE}} + clang_analyzer_eval(myns::embed_cls().fecl(1) == -6); // expected-warning{{TRUE}} + clang_analyzer_eval(mycls::embed_cls2().fecl2(0) == -11); // expected-warning{{TRUE}} + + clang_analyzer_eval(chns::chf1(4) == 12); // expected-warning{{TRUE}} + clang_analyzer_eval(fun_using_anon_struct(8) == 8); // expected-warning{{TRUE}} + + clang_analyzer_eval(other_macro_diag(1) == 1); // expected-warning{{TRUE}} + // expected-warning@Inputs/ctu-other.cpp:93{{REACHABLE}} + MACRODIAG(); // expected-warning{{REACHABLE}} +} diff --git a/test/Analysis/ctu-unknown-parts-in-triples.cpp b/test/Analysis/ctu-unknown-parts-in-triples.cpp index 5e643c164dd7..8009937c909d 100644 --- a/test/Analysis/ctu-unknown-parts-in-triples.cpp +++ b/test/Analysis/ctu-unknown-parts-in-triples.cpp @@ -5,7 +5,7 @@ // RUN: mkdir -p %t/ctudir // RUN: %clang_cc1 -triple x86_64-pc-linux-gnu \ // RUN: -emit-pch -o %t/ctudir/ctu-other.cpp.ast %S/Inputs/ctu-other.cpp -// RUN: cp %S/Inputs/ctu-other.cpp.externalDefMap.txt %t/ctudir/externalDefMap.txt +// RUN: cp %S/Inputs/ctu-other.cpp.externalDefMap.ast-dump.txt %t/ctudir/externalDefMap.txt // RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux-gnu \ // RUN: -analyzer-checker=core,debug.ExprInspection \ // RUN: -analyzer-config experimental-enable-naive-ctu-analysis=true \ diff --git a/unittests/CrossTU/CrossTranslationUnitTest.cpp b/unittests/CrossTU/CrossTranslationUnitTest.cpp index b4f1447762b9..0333dfd16238 100644 --- a/unittests/CrossTU/CrossTranslationUnitTest.cpp +++ b/unittests/CrossTU/CrossTranslationUnitTest.cpp @@ -11,6 +11,7 @@ #include "clang/AST/ASTConsumer.h" #include "clang/Frontend/FrontendAction.h" #include "clang/Tooling/Tooling.h" +#include "llvm/ADT/Optional.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" #include "llvm/Support/ToolOutputFile.h" @@ -73,7 +74,8 @@ class CTUASTConsumer : public clang::ASTConsumer { // Load the definition from the AST file. llvm::Expected NewFDorError = handleExpected( - CTU.getCrossTUDefinition(FD, "", IndexFileName, false, ImportLimit), + CTU.getCrossTUDefinition(FD, "", IndexFileName, false, ImportLimit, + llvm::Optional{}), []() { return nullptr; }, [](IndexError &) {}); if (NewFDorError) { @@ -136,7 +138,7 @@ TEST(CrossTranslationUnit, IndexFormatCanBeParsed) { IndexFile.os().flush(); EXPECT_TRUE(llvm::sys::fs::exists(IndexFileName)); llvm::Expected> IndexOrErr = - parseCrossTUIndex(IndexFileName, ""); + parseCrossTUIndex(IndexFileName, "", llvm::Optional{}); EXPECT_TRUE((bool)IndexOrErr); llvm::StringMap ParsedIndex = IndexOrErr.get(); for (const auto &E : Index) { @@ -161,7 +163,7 @@ TEST(CrossTranslationUnit, CTUDirIsHandledCorrectly) { IndexFile.os().flush(); EXPECT_TRUE(llvm::sys::fs::exists(IndexFileName)); llvm::Expected> IndexOrErr = - parseCrossTUIndex(IndexFileName, "/ctudir"); + parseCrossTUIndex(IndexFileName, "/ctudir", llvm::Optional{}); EXPECT_TRUE((bool)IndexOrErr); llvm::StringMap ParsedIndex = IndexOrErr.get(); EXPECT_EQ(ParsedIndex["a"], "/ctudir/b/c/d");