Merge remote-tracking branch 'origin/master' into merge-2.104

.cirrus.yml

@@ -67,7 +67,7 @@ common_steps_template: &COMMON_STEPS_TEMPLATE
         excludes+='|^druntime-test-exceptions-debug$'
       elif [[ "$CI_OS-$CI_ARCH" == "osx-arm64" ]]; then
         # FIXME: sporadic segfaults/bus errors with enabled optimizations
-        excludes+='|^core.thread.fiber-shared$'
+        excludes+='|^core.thread.fiber(-shared)?$'
       fi
       ctest -j$PARALLELISM --output-on-failure -E "$excludes"
 

.github/actions/3-build-cross/action.yml

@@ -145,4 +145,4 @@ runs:
           ${{ inputs.cmake_flags }}
           ${{ inputs.with_pgo == 'true' && '-DDFLAGS_LDC=-fprofile-use=../pgo-ldc/merged.profdata' || '' }}
           ${{ env.CROSS_CMAKE_FLAGS }}
-        build_targets: ldc2 ldmd2 ldc-build-runtime ldc-profdata ldc-prune-cache timetrace2txt
+        build_targets: ldc2 ldmd2 ldc-build-runtime ldc-build-plugin ldc-profdata ldc-prune-cache timetrace2txt

.github/actions/5-install/action.yml

@@ -22,6 +22,7 @@ runs:
         else
           mkdir -p install/bin
           cp build-cross/bin/{ldc2,ldmd2,ldc-build-runtime,ldc-profdata,ldc-prune-cache,timetrace2txt} install/bin/
+          cp build-cross/bin/ldc-build-plugin install/bin/ || true
           cp -R build-cross-libs/lib install/
           cp build-cross/lib/{libldc_rt.*,libLTO-ldc.dylib,LLVMgold-ldc.so} install/lib/ || true
           mkdir install/etc

CHANGELOG.md

@@ -1,13 +1,25 @@
 # LDC master
 
+#### Big news
+
+#### Platform support
+
+#### Bug fixes
+
+# LDC 1.33.0 (2023-07-23)
+
 #### Big news
 - Frontend, druntime and Phobos are at version [2.103.1](https://dlang.org/changelog/2.103.0.html), incl. new command-line option `-verror-supplements`. (#4345)
+- The `--plugin` commandline option now also accepts semantic analysis plugins. Semantic analysis plugins are recognized by exporting the symbol: `extern(C) void runSemanticAnalysis(Module m)`. The plugin's `runSemanticAnalysis` function is called for each module, after all other semantic analysis steps (also after DCompute SemA), just before object codegen. (#4430)
+- New tool `ldc-build-plugin` that helps compiling user plugins. It downloads the correct LDC source version (if it's not already available), and calls LDC with the correct commandline flags to build a plugin. (#4430)
 - New commandline option `-femit-local-var-lifetime` that enables variable lifetime (scope) annotation to LLVM IR codegen. Lifetime annotation enables stack memory reuse for local variables with non-overlapping scope. (#4395)
 - C files are now automatically preprocessed using the external C compiler (configurable via `-gcc` or the `CC` environment variable, and `-Xcc` for extra flags). Extra preprocessor flags (e.g., include dirs and manual defines) can be added via new command-line option `-P`. (#4417)
   - Windows: If `clang-cl.exe` is on `PATH`, it is preferred over Microsoft's `cl.exe` by default (e.g., to avoid printing the C source file name to stderr during preprocessing).
 - Less pedantic checks for conflicting C(++) function declarations when compiling multiple modules to a single object file ('Error: Function type does not match previously declared function with the same mangled name'). The error now only appears if an object file actually references multiple conflicting functions. (#4420)
+- New command-line option `--fcf-protection`, which enables Intel's Control-Flow Enforcement Technology (CET). (#4437)
 
 #### Platform support
+- Supports LLVM 9.0 - 15.0.
 
 #### Bug fixes
 - Handle potential lambda mangle collisions across separately compiled object files (and the linker then silently picking an arbitrary implementation). Lambdas (and their nested global variables) are now internal to each referencing object file (`static` linkage in C). (#4415)

CMakeLists.txt

@@ -249,17 +249,6 @@ if(CMAKE_COMPILER_IS_GNUCXX OR (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang"))
     endif()
 endif()
 
-if(NOT WIN32 AND NOT CYGWIN)
-    # Unify symbol visibility with LLVM to silence linker warning "direct access in function X to global
-    # weak symbol Y means the weak symbol cannot be overridden at runtime. This was likely caused by
-    # different translation units being compiled with different visibility settings."
-    # See LLVM's cmake/modules/HandleLLVMOptions.cmake.
-    check_cxx_compiler_flag("-fvisibility-inlines-hidden" SUPPORTS_FVISIBILITY_INLINES_HIDDEN_FLAG)
-    if (${SUPPORTS_FVISIBILITY_INLINES_HIDDEN_FLAG})
-        append("-fvisibility-inlines-hidden" LDC_CXXFLAGS)
-    endif()
-endif()
-
 if(MSVC)
     # Remove flags here, for exceptions and RTTI.
     # CL.EXE complains to override flags like "/GR /GR-".
@@ -683,10 +672,26 @@ if(LDC_ENABLE_PLUGINS)
 
         if(LINKER_ACCEPTS_EXPORT_DYNAMIC_FLAG)
             set(LDC_LINKERFLAG_LIST "${LDC_LINKERFLAG_LIST};-Wl,--export-dynamic")
+        else()
+            message(WARNING "Linker does not accept --export-dynamic, user plugins may give missing symbol errors upon load")
         endif()
     endif()
 endif()
 message(STATUS "-- Building LDC with plugin support (LDC_ENABLE_PLUGINS): ${LDC_ENABLE_PLUGINS}")
+message(STATUS "-- Linking LDC with flags: ${ALTERNATIVE_MALLOC_O};${LDC_LINKERFLAG_LIST}")
+
+if(NOT WIN32 AND NOT CYGWIN)
+    # Unify symbol visibility with LLVM to silence linker warning "direct access in function X to global
+    # weak symbol Y means the weak symbol cannot be overridden at runtime. This was likely caused by
+    # different translation units being compiled with different visibility settings."
+    # See LLVM's cmake/modules/HandleLLVMOptions.cmake.
+    check_cxx_compiler_flag("-fvisibility-inlines-hidden" SUPPORTS_FVISIBILITY_INLINES_HIDDEN_FLAG)
+    if (LDC_ENABLE_PLUGINS AND NOT APPLE)
+        # For plugins, we shouldn't apply this flag because it hides the inline methods of e.g. Visitor. On macOS it's OK to add.
+    elseif (${SUPPORTS_FVISIBILITY_INLINES_HIDDEN_FLAG})
+        append("-fvisibility-inlines-hidden" LDC_CXXFLAGS)
+    endif()
+endif()
 
 build_d_executable(
     "${LDC_EXE}"

cmake/Modules/BuildDExecutable.cmake

@@ -25,6 +25,7 @@ endmacro()
 # - DFLAGS_BASE
 # - LDC_LINK_MANUALLY
 # - D_LINKER_ARGS
+# - LDC_ENABLE_PLUGINS
 function(build_d_executable target_name output_exe d_src_files compiler_args linker_args extra_compile_deps link_deps compile_separately)
     set(dflags "${D_COMPILER_FLAGS} ${DFLAGS_BASE} ${compiler_args}")
     if(UNIX)
@@ -40,7 +41,9 @@ function(build_d_executable target_name output_exe d_src_files compiler_args lin
         # Compile all D modules to a single object.
         set(object_file ${PROJECT_BINARY_DIR}/obj/${target_name}${CMAKE_CXX_OUTPUT_EXTENSION})
         # Default to -linkonce-templates with LDMD host compiler, to speed-up optimization.
-        if("${D_COMPILER_ID}" STREQUAL "LDMD")
+        if("${target_name}" STREQUAL "ldc2" AND LDC_ENABLE_PLUGINS)
+            # For plugin support we need ldc2's symbols to be global, don't use -linkonce-templates.
+        elseif("${D_COMPILER_ID}" STREQUAL "LDMD")
             set(dflags -linkonce-templates ${dflags})
         endif()
         add_custom_command(

dmd/common/string.d

@@ -136,8 +136,16 @@ but is guaranteed to follow it.
 version(Windows) wchar[] toWStringz(const(char)[] narrow, ref SmallBuffer!wchar buffer) nothrow
 {
     import core.sys.windows.winnls : CP_ACP, MultiByteToWideChar;
+
+version (IN_LLVM)
+{
+    import dmd.root.filename : CodePage;
+}
+else
+{
     // assume filenames encoded in system default Windows ANSI code page
     enum CodePage = CP_ACP;
+}
 
     if (narrow is null)
         return null;

dmd/root/filename.d

@@ -44,7 +44,7 @@ version (Windows)
 
 version (IN_LLVM)
 {
-    private enum CodePage = CP_UTF8;
+    enum CodePage = CP_UTF8;
 }
 else
 {

driver/cl_options_instrumentation.cpp

@@ -84,6 +84,19 @@ llvm::StringRef getXRayInstructionThresholdString() {
   return thresholdString;
 }
 
+cl::opt<CFProtectionType> fCFProtection(
+    "fcf-protection",
+    cl::desc("Instrument control-flow architecture protection"), cl::ZeroOrMore,
+    cl::ValueOptional,
+    cl::values(clEnumValN(CFProtectionType::None, "none", ""),
+               clEnumValN(CFProtectionType::Branch, "branch", ""),
+               clEnumValN(CFProtectionType::Return, "return", ""),
+               clEnumValN(CFProtectionType::Full, "full", ""),
+               clEnumValN(CFProtectionType::Full, "",
+                          "") // default to "full" if no argument specified
+               ),
+    cl::init(CFProtectionType::None));
+
 void initializeInstrumentationOptionsFromCmdline(const llvm::Triple &triple) {
   if (ASTPGOInstrGenFile.getNumOccurrences() > 0) {
     pgoMode = PGO_ASTBasedInstr;
@@ -110,6 +123,14 @@ void initializeInstrumentationOptionsFromCmdline(const llvm::Triple &triple) {
 
   if (dmdFunctionTrace)
     global.params.trace = true;
+
+  // fcf-protection is only valid for X86
+  if (fCFProtection != CFProtectionType::None &&
+      !(triple.getArch() == llvm::Triple::x86 ||
+        triple.getArch() == llvm::Triple::x86_64)) {
+    error(Loc(), "option '--fcf-protection' cannot be specified on this target "
+                 "architecture");
+  }
 }
 
 } // namespace opts

driver/cl_options_instrumentation.h

@@ -29,6 +29,9 @@ extern cl::opt<bool> instrumentFunctions;
 extern cl::opt<bool> fXRayInstrument;
 llvm::StringRef getXRayInstructionThresholdString();
 
+enum class CFProtectionType { None = 0, Branch = 1, Return = 2, Full = 3 };
+extern cl::opt<CFProtectionType> fCFProtection;
+
 /// This initializes the instrumentation options, and checks the validity of the
 /// commandline flags. targetTriple should be initialized before calling this.
 /// It should be called only once.

driver/plugins.cpp

@@ -9,6 +9,9 @@
 //
 // Implements functionality related to plugins (`-plugin=...`).
 //
+// Note: plugins can be LLVM-plugins (to be registered with the pass manager)
+// or dlang-plugins for semantic analysis.
+//
 //===----------------------------------------------------------------------===//
 
 #include "driver/plugins.h"
@@ -17,6 +20,7 @@
 
 #include "dmd/errors.h"
 #include "dmd/globals.h"
+#include "dmd/module.h"
 #include "llvm/Passes/PassBuilder.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/DynamicLibrary.h"
@@ -36,63 +40,120 @@ cl::list<std::string> pluginFiles("plugin", cl::CommaSeparated,
                                   cl::desc("Pass plugins to load."),
                                   cl::value_desc("dynamic_library.so,lib2.so"));
 
+struct SemaPlugin {
+  llvm::sys::DynamicLibrary library;
+  void (*runSemanticAnalysis)(Module *);
+
+  SemaPlugin(const llvm::sys::DynamicLibrary &library,
+             void (*runSemanticAnalysis)(Module *))
+      : library(library), runSemanticAnalysis(runSemanticAnalysis) {}
+};
+
+llvm::SmallVector<SemaPlugin, 1> sema_plugins;
+
 } // anonymous namespace
 
-#if LDC_LLVM_VER >= 1400
+// Tries to load plugin as SemanticAnalysis. Returns true on 'success', i.e. no
+// further attempts needed.
+bool loadSemanticAnalysisPlugin(const std::string &filename) {
+  std::string errorString;
+  auto library = llvm::sys::DynamicLibrary::getPermanentLibrary(
+      filename.c_str(), &errorString);
+  if (!library.isValid()) {
+    error(Loc(), "Error loading plugin '%s': %s", filename.c_str(),
+          errorString.c_str());
+    return true; // No success, but no need to try loading again as LLVM plugin.
+  }
 
-namespace {
-llvm::SmallVector<llvm::PassPlugin, 1> plugins;
+  // SemanticAnalysis plugins need to export the `runSemanticAnalysis` function.
+  void *runSemanticAnalysisFnPtr =
+      library.getAddressOfSymbol("runSemanticAnalysis");
+
+  // If the symbol isn't found, this is probably an LLVM plugin.
+  if (!runSemanticAnalysisFnPtr)
+    return false;
+
+  sema_plugins.emplace_back(
+      library, reinterpret_cast<void (*)(Module *)>(runSemanticAnalysisFnPtr));
+  return true;
 }
-/// Loads all plugins for the new pass manager. These plugins will need to be
-/// added When building the optimization pipeline.
-void loadAllPluginsNewPM() {
-  for (auto &filename : pluginFiles) {
-    auto plugin = llvm::PassPlugin::Load(filename);
-    if (!plugin) {
-      error(Loc(), "Error loading plugin '%s': %s", filename.c_str(),
-            llvm::toString(plugin.takeError()).c_str());
-      continue;
-    }
-    plugins.emplace_back(plugin.get());
+
+/// Loads plugin for the legacy pass manager. The static constructor of
+/// the plugin should take care of the plugins registering themself with the
+/// rest of LDC/LLVM.
+void loadLLVMPluginLegacyPM(const std::string &filename) {
+  std::string errorString;
+  if (llvm::sys::DynamicLibrary::LoadLibraryPermanently(filename.c_str(),
+                                                        &errorString)) {
+    error(Loc(), "Error loading plugin '%s': %s", filename.c_str(),
+          errorString.c_str());
   }
 }
-void registerAllPluginsWithPassBuilder(llvm::PassBuilder &PB) {
-  for (auto &plugin : plugins) {
-    plugin.registerPassBuilderCallbacks(PB);
+
+#if LDC_LLVM_VER >= 1400
+
+namespace {
+llvm::SmallVector<llvm::PassPlugin, 1> llvm_plugins;
+
+/// Loads plugin for the new pass manager. The plugin will need to be
+/// added explicitly when building the optimization pipeline.
+void loadLLVMPluginNewPM(const std::string &filename) {
+
+  auto plugin = llvm::PassPlugin::Load(filename);
+  if (!plugin) {
+    error(Loc(), "Error loading plugin '%s': %s", filename.c_str(),
+          llvm::toString(plugin.takeError()).c_str());
+    return;
   }
+  llvm_plugins.emplace_back(plugin.get());
 }
 
+} // anonymous namespace
+
 #endif // LDC_LLVM_VER >= 1400
 
-/// Loads all plugins for the legacy pass manaager. The static constructor of
-/// each plugin should take care of the plugins registering themself with the
-/// rest of LDC/LLVM.
-void loadAllPluginsLegacyPM() {
+void loadLLVMPlugin(const std::string &filename) {
+#if LDC_LLVM_VER >= 1400
+  if (opts::isUsingLegacyPassManager())
+    loadLLVMPluginLegacyPM(filename);
+  else
+    loadLLVMPluginNewPM(filename);
+#else
+  loadLLVMPluginLegacyPM(filename);
+#endif
+}
+
+void loadAllPlugins() {
   for (auto &filename : pluginFiles) {
-    std::string errorString;
-    if (llvm::sys::DynamicLibrary::LoadLibraryPermanently(filename.c_str(),
-                                                          &errorString)) {
-      error(Loc(), "Error loading plugin '%s': %s", filename.c_str(),
-            errorString.c_str());
-    }
+    // First attempt to load plugin as SemanticAnalysis plugin. If unsuccesfull,
+    // load as LLVM plugin.
+    auto success = loadSemanticAnalysisPlugin(filename);
+    if (!success)
+      loadLLVMPlugin(filename);
   }
 }
 
+void registerAllPluginsWithPassBuilder(llvm::PassBuilder &PB) {
 #if LDC_LLVM_VER >= 1400
-void loadAllPlugins() {
-  if (opts::isUsingLegacyPassManager())
-    loadAllPluginsLegacyPM();
-  else
-    loadAllPluginsNewPM();
-}
-#else
-void loadAllPlugins() { loadAllPluginsLegacyPM(); }
-void registerAllPluginsWithPassBuilder(llvm::PassBuilder &) {}
+  for (auto &plugin : llvm_plugins) {
+    plugin.registerPassBuilderCallbacks(PB);
+  }
 #endif
+}
+
+void runAllSemanticAnalysisPlugins(Module *m) {
+  for (auto &plugin : sema_plugins) {
+    assert(plugin.runSemanticAnalysis);
+    plugin.runSemanticAnalysis(m);
+  }
+}
 
 #else // #if LDC_ENABLE_PLUGINS
 
+class Module;
+
 void loadAllPlugins() {}
 void registerAllPluginsWithPassBuilder(llvm::PassBuilder &) {}
+void runAllSemanticAnalysisPlugins(Module *m) {}
 
 #endif // LDC_ENABLE_PLUGINS

gen/llvmhelpers.cpp

@@ -1188,7 +1188,7 @@ LLConstant *DtoConstExpInit(const Loc &loc, Type *targetType, Expression *exp) {
       val = llvm::ConstantArray::get(at, elements);
     }
 
-    (void)numTotalVals;
+    (void)numTotalVals; (void) product; // Silence unused variable warning when assert is disabled.
     assert(product == numTotalVals);
     return val;
   }