shader compilation rework

.clang-tidy

@@ -18,7 +18,8 @@ Checks: >
   -readability-redundant-string-init,
   -readability-identifier-length,
   -readability-avoid-const-params-in-decls,
-  -readability-uppercase-literal-suffix
+  -readability-uppercase-literal-suffix,
+  -bugprone-easily-swappable-parameters
 CheckOptions:
   - { key: readability-identifier-naming.NamespaceCase,          value: lower_case }
   - { key: readability-identifier-naming.ClassCase,              value: CamelCase  }

include/merian/io/file_loader.hpp

@@ -1,6 +1,8 @@
 #pragma once
 
+#include "merian/utils/string.hpp"
 #include <filesystem>
+#include <fstream>
 #include <optional>
 #include <set>
 #include <spdlog/spdlog.h>
@@ -14,9 +16,32 @@ class FileLoader {
     static bool exists(const std::filesystem::path& path,
                        std::filesystem::file_status file_status = std::filesystem::file_status{});
 
+    template <typename T> static std::vector<T> load_file(const std::filesystem::path& path) {
+        if (!exists(path)) {
+            throw std::runtime_error{
+                fmt::format("failed to load {} (does not exist)", path.string())};
+        }
+
+        // Open the stream to 'lock' the file.
+        std::ifstream f(path, std::ios::in | std::ios::binary);
+        const std::size_t size = std::filesystem::file_size(path);
+
+        if (size % sizeof(T)) {
+            SPDLOG_WARN("loading {} B of data into a vector quantized to {} B", size, sizeof(T));
+        }
+
+        std::vector<T> result((size + sizeof(T) - 1) / sizeof(T), {});
+        f.read((char*)result.data(), (std::streamsize)size);
+
+        SPDLOG_DEBUG("load {} of data from {}", format_size(size), path.string());
+
+        return result;
+    }
+
     static std::string load_file(const std::filesystem::path& path);
 
-    static std::optional<std::filesystem::path> search_cwd_parents(const std::filesystem::path& path);
+    static std::optional<std::filesystem::path>
+    search_cwd_parents(const std::filesystem::path& path);
 
   public:
     FileLoader(const std::set<std::filesystem::path>& search_paths = {"./"})

include/merian/utils/filesystem.hpp

@@ -0,0 +1,39 @@
+#pragma once
+
+#include <string>
+#include <unistd.h>
+
+#ifdef _WIN32
+#include <cstdio>
+#endif
+
+namespace merian {
+
+inline std::string temporary_file() {
+#ifdef _WIN32
+    return std::tmpnam(nullptr);
+#else
+    // std::tmpnam is deprecated
+    const char* tem_dir_name = std::getenv("TMPDIR");
+    std::string tmp_file_name_template = (tem_dir_name != nullptr) ? tem_dir_name : "";
+#ifdef P_tmpdir
+    if (tmp_file_name_template.empty()) {
+        tmp_file_name_template = P_tmpdir;
+    }
+#endif
+    if (tmp_file_name_template.empty()) {
+        tmp_file_name_template = "/tmp";
+    }
+
+    tmp_file_name_template += "/merianXXXXXX";
+    const int fd = mkstemp(const_cast<char*>(tmp_file_name_template.c_str()));
+    if (fd > 0) {
+        // immediately close again...
+        close(fd);
+    }
+
+    return tmp_file_name_template;
+#endif
+}
+
+} // namespace merian

include/merian/vk/context.hpp

@@ -2,6 +2,8 @@
 
 #include "merian/io/file_loader.hpp"
 #include "merian/utils/concurrent/thread_pool.hpp"
+#include <map>
+#include <ranges>
 #include <spdlog/logger.h>
 
 #include <typeindex>
@@ -125,6 +127,7 @@ class Context : public std::enable_shared_from_this<Context> {
                                  std::string filter_device_name);
     void find_queues();
     void create_device_and_queues(uint32_t preffered_number_compute_queues);
+    void prepare_shader_include_defines();
 
   private: // Helper
     void extensions_check_instance_layer_support();
@@ -179,12 +182,14 @@ class Context : public std::enable_shared_from_this<Context> {
 
     bool instance_extension_enabled(const std::string& name) const;
 
-    auto get_context_extensions() const;
-
     const std::vector<const char*>& get_enabled_device_extensions() const;
 
     const std::vector<const char*>& get_enabled_instance_extensions() const;
 
+    const std::vector<std::string>& get_default_shader_include_paths() const;
+
+    const std::map<std::string, std::string>& get_default_shader_macro_definitions() const;
+
   private:
     std::unordered_map<std::type_index, std::shared_ptr<Extension>> extensions;
 
@@ -257,6 +262,10 @@ class Context : public std::enable_shared_from_this<Context> {
     std::weak_ptr<CommandPool> cmd_pool_T;
     // Convenience command pool for compute (can be nullptr in very rare occasions)
     std::weak_ptr<CommandPool> cmd_pool_C;
+
+
+    std::vector<std::string> default_shader_include_paths;
+    std::map<std::string, std::string> default_shader_macro_definitions;
 };
 
 } // namespace merian

include/merian/vk/extension/extension.hpp

@@ -3,6 +3,7 @@
 #include "merian/vk/context.hpp"
 #include <spdlog/spdlog.h>
 #include <vector>
+#include <map>
 
 namespace merian {
 
@@ -114,7 +115,7 @@ class Extension {
     // return strings that should be defined when compiling shaders with Merians shader compiler.
     // Note that device and instance extensions are automatically defined as 
     // MERIAN_DEVICE_EXT_ENABLE_<NAME> and MERIAN_INSTANCE_EXT_ENABLE_<NAME>
-    virtual std::vector<std::string> shader_defines() {
+    virtual std::map<std::string, std::string> shader_macro_definitions() {
         return {};
     }
 

include/merian/vk/shader/shader_compiler.hpp

@@ -14,34 +14,84 @@ namespace merian {
 // A compiler for shaders.
 //
 // Include paths for the merian-nodes library and context extensions must be automatically added.
+class ShaderCompiler;
+using ShaderCompilerHandle = std::shared_ptr<ShaderCompiler>;
+
 class ShaderCompiler {
   public:
     class compilation_failed : public std::runtime_error {
       public:
         compilation_failed(const std::string& what) : std::runtime_error(what) {}
     };
 
+    inline static const std::map<std::string, vk::ShaderStageFlagBits> EXTENSION_SHADER_STAGE_MAP =
+        {
+            {".vert", vk::ShaderStageFlagBits::eVertex},
+            {".tesc", vk::ShaderStageFlagBits::eTessellationControl},
+            {".tese", vk::ShaderStageFlagBits::eTessellationEvaluation},
+            {".geom", vk::ShaderStageFlagBits::eGeometry},
+            {".frag", vk::ShaderStageFlagBits::eFragment},
+            {".comp", vk::ShaderStageFlagBits::eCompute},
+            {".mesh", vk::ShaderStageFlagBits::eMeshEXT},
+            {".task", vk::ShaderStageFlagBits::eTaskEXT},
+            {".rgen", vk::ShaderStageFlagBits::eRaygenKHR},
+            {".rint", vk::ShaderStageFlagBits::eIntersectionKHR},
+            {".rahit", vk::ShaderStageFlagBits::eAnyHitKHR},
+            {".rchit", vk::ShaderStageFlagBits::eClosestHitKHR},
+            {".rmiss", vk::ShaderStageFlagBits::eMissKHR},
+            {".rcall", vk::ShaderStageFlagBits::eCallableKHR},
+    };
+
+    inline static const std::map<vk::ShaderStageFlagBits, std::string> SHADER_STAGE_EXTENSION_MAP =
+        {
+            {vk::ShaderStageFlagBits::eVertex, ".vert"},
+            {vk::ShaderStageFlagBits::eTessellationControl, ".tesc"},
+            {vk::ShaderStageFlagBits::eTessellationEvaluation, ".tese"},
+            {vk::ShaderStageFlagBits::eGeometry, ".geom"},
+            {vk::ShaderStageFlagBits::eFragment, ".frag"},
+            {vk::ShaderStageFlagBits::eCompute, ".comp"},
+            {vk::ShaderStageFlagBits::eMeshEXT, ".mesh"},
+            {vk::ShaderStageFlagBits::eTaskEXT, ".task"},
+            {vk::ShaderStageFlagBits::eRaygenKHR, ".rgen"},
+            {vk::ShaderStageFlagBits::eIntersectionKHR, ".rint"},
+            {vk::ShaderStageFlagBits::eAnyHitKHR, ".rahit"},
+            {vk::ShaderStageFlagBits::eClosestHitKHR, ".rchit"},
+            {vk::ShaderStageFlagBits::eMissKHR, ".rmiss"},
+            {vk::ShaderStageFlagBits::eCallableKHR, ".rcall"},
+    };
+
   public:
-    ShaderCompiler(const std::vector<std::string>& user_include_paths = {},
-                   const std::map<std::string, std::string>& user_macro_definitions = {})
-        : include_paths(user_include_paths), macro_definitions(user_macro_definitions) {
-        // search merian-shaders
-
-        // add macro definitions from context extensions and enabled instance and device extensions.
-    }
+    // Returns any of the available shader compilers. Returns nullptr if none is available.
+    static ShaderCompilerHandle
+    get(const ContextHandle& context,
+        const std::vector<std::string>& user_include_paths = {},
+        const std::map<std::string, std::string>& user_macro_definitions = {});
+
+    ShaderCompiler(const ContextHandle& context,
+                   const std::vector<std::string>& user_include_paths = {},
+                   const std::map<std::string, std::string>& user_macro_definitions = {});
 
     virtual ~ShaderCompiler() = 0;
 
+    // ------------------------------------------------
+
     // Attempt to guess the shader_kind from the file extension if shader_kind = std::nullopt.
     //
     // May throw compilation_failed.
-    std::vector<uint32_t>
+    virtual std::vector<uint32_t>
     compile_glsl(const std::filesystem::path& path,
                  const std::optional<vk::ShaderStageFlagBits> optional_shader_kind = std::nullopt) {
         return compile_glsl(FileLoader::load_file(path), path.string(),
                             optional_shader_kind.value_or(guess_kind(path)));
     }
 
+    // May throw compilation_failed.
+    virtual std::vector<uint32_t> compile_glsl(const std::string& source,
+                                               const std::string& source_name,
+                                               const vk::ShaderStageFlagBits shader_kind) = 0;
+
+    // ------------------------------------------------
+
     ShaderModuleHandle compile_glsl_to_shadermodule(
         const ContextHandle& context,
         const std::filesystem::path& path,
@@ -59,10 +109,7 @@ class ShaderCompiler {
             context, compile_glsl(source, source_name, shader_kind), shader_kind);
     }
 
-    // May throw compilation_failed.
-    virtual std::vector<uint32_t> compile_glsl(const std::string& source,
-                                               const std::string& source_name,
-                                               const vk::ShaderStageFlagBits shader_kind) = 0;
+    // ------------------------------------------------
 
     const std::vector<std::string>& get_include_paths() const {
         return include_paths;
@@ -72,6 +119,8 @@ class ShaderCompiler {
         return macro_definitions;
     }
 
+    virtual bool available() const = 0;
+
   private:
     static vk::ShaderStageFlagBits guess_kind(const std::filesystem::path& path) {
         std::string extension;
@@ -83,23 +132,8 @@ class ShaderCompiler {
             extension = path.extension().string();
         }
 
-        if (extension == ".vert") {
-            return vk::ShaderStageFlagBits::eVertex;
-        }
-        if (extension == ".tesc") {
-            return vk::ShaderStageFlagBits::eTessellationControl;
-        }
-        if (extension == ".tese") {
-            return vk::ShaderStageFlagBits::eTessellationEvaluation;
-        }
-        if (extension == ".geom") {
-            return vk::ShaderStageFlagBits::eGeometry;
-        }
-        if (extension == ".frag") {
-            return vk::ShaderStageFlagBits::eFragment;
-        }
-        if (extension == ".comp") {
-            return vk::ShaderStageFlagBits::eCompute;
+        if (EXTENSION_SHADER_STAGE_MAP.contains(extension)) {
+            return EXTENSION_SHADER_STAGE_MAP.at(extension);
         }
 
         throw compilation_failed{
@@ -109,6 +143,5 @@ class ShaderCompiler {
     std::vector<std::string> include_paths;
     std::map<std::string, std::string> macro_definitions;
 };
-using ShaderCompilerHandle = std::shared_ptr<ShaderCompiler>;
 
 } // namespace merian

include/merian/vk/shader/shader_compiler_shaderc.hpp

@@ -3,21 +3,16 @@
 #include "merian/vk/shader/shader_compiler.hpp"
 #include <map>
 
-#ifdef __has_include
-#if !__has_include(<shaderc/shaderc.hpp>)
-static_assert(false, "shaderc is required for ShadercCompiler");
-#else
-#include <shaderc/shaderc.hpp>
-#endif
-#else
+#ifdef MERIAN_SHADERC_FOUND
 #include <shaderc/shaderc.hpp>
 #endif
 
 namespace merian {
 
 class ShadercCompiler : public ShaderCompiler {
   public:
-    ShadercCompiler(const std::vector<std::string>& include_paths = {},
+    ShadercCompiler(const ContextHandle& context,
+                    const std::vector<std::string>& include_paths = {},
                     const std::map<std::string, std::string>& macro_definitions = {});
 
     ~ShadercCompiler();
@@ -26,9 +21,13 @@ class ShadercCompiler : public ShaderCompiler {
                                        const std::string& source_name,
                                        const vk::ShaderStageFlagBits shader_kind) override;
 
+    bool available() const override;
+
   private:
+#ifdef MERIAN_SHADERC_FOUND
     shaderc::Compiler shader_compiler;
     shaderc::CompileOptions compile_options;
+#endif
 };
 
 } // namespace merian

include/merian/vk/shader/shader_compiler_glsllangValidator.hpp → include/merian/vk/shader/shader_compiler_system_glslangValidator.hpp

@@ -6,17 +6,24 @@
 namespace merian {
 
 // Uses glslangValidator executable to compile shaders.
-class GLSLLangValidatorCompiler : public ShaderCompiler {
+class SystemGlslangValidatorCompiler : public ShaderCompiler {
   public:
     // Include paths for the merian-nodes library are automatically added
-    GLSLLangValidatorCompiler(const std::vector<std::string>& include_paths = {},
-                              const std::map<std::string, std::string>& macro_definitions = {});
+    SystemGlslangValidatorCompiler(
+        const ContextHandle& context,
+        const std::vector<std::string>& include_paths = {},
+        const std::map<std::string, std::string>& macro_definitions = {});
 
-    ~GLSLLangValidatorCompiler();
+    ~SystemGlslangValidatorCompiler();
 
     std::vector<uint32_t> compile_glsl(const std::string& source,
                                        const std::string& source_name,
                                        const vk::ShaderStageFlagBits shader_kind) override;
+
+    bool available() const override;
+
+  private:
+    const ContextHandle context;
 };
 
 } // namespace merian

include/merian/vk/shader/shader_compiler_system_glslc.hpp

@@ -0,0 +1,28 @@
+#pragma once
+
+#include "merian/vk/shader/shader_compiler.hpp"
+#include <map>
+
+namespace merian {
+
+// Uses shaderc executable to compile shaders.
+class SystemGlslcCompiler : public ShaderCompiler {
+  public:
+    // Include paths for the merian-nodes library are automatically added
+    SystemGlslcCompiler(const ContextHandle& context,
+                        const std::vector<std::string>& include_paths = {},
+                        const std::map<std::string, std::string>& macro_definitions = {});
+
+    ~SystemGlslcCompiler();
+
+    std::vector<uint32_t> compile_glsl(const std::string& source,
+                                       const std::string& source_name,
+                                       const vk::ShaderStageFlagBits shader_kind) override;
+
+    bool available() const override;
+
+  private:
+    const ContextHandle context;
+};
+
+} // namespace merian