From e58381799dec4030c57bd3dcd30c0605be21469a Mon Sep 17 00:00:00 2001 From: Lucas Alber Date: Wed, 6 Nov 2024 17:57:40 +0100 Subject: [PATCH] shader compilation rework --- .clang-tidy | 3 +- include/merian/io/file_loader.hpp | 27 +++++- include/merian/utils/filesystem.hpp | 39 ++++++++ include/merian/vk/context.hpp | 13 ++- include/merian/vk/extension/extension.hpp | 3 +- include/merian/vk/shader/shader_compiler.hpp | 93 +++++++++++++------ .../vk/shader/shader_compiler_shaderc.hpp | 15 ++- ...ader_compiler_system_glslangValidator.hpp} | 15 ++- .../shader/shader_compiler_system_glslc.hpp | 28 ++++++ include/merian/vk/shader/shader_module.hpp | 2 +- meson.build | 13 ++- .../nodes/shadertoy/shadertoy.cpp | 23 +++-- src/merian/io/file_loader.cpp | 4 +- src/merian/meson.build | 4 + src/merian/vk/context.cpp | 65 +++++++++++-- src/merian/vk/shader/meson.build | 2 + src/merian/vk/shader/shader_compiler.cpp | 45 ++++++++- .../vk/shader/shader_compiler_shaderc.cpp | 11 ++- .../shader/shader_compiler_shaderc_stub.cpp | 26 ++++++ ...hader_compiler_system_glslangValidator.cpp | 79 ++++++++++++++++ .../shader/shader_compiler_system_glslc.cpp | 80 ++++++++++++++++ src/merian/vk/shader/shader_module.cpp | 2 +- subprojects/subprocess.wrap | 11 +++ 23 files changed, 523 insertions(+), 80 deletions(-) create mode 100644 include/merian/utils/filesystem.hpp rename include/merian/vk/shader/{shader_compiler_glsllangValidator.hpp => shader_compiler_system_glslangValidator.hpp} (55%) create mode 100644 include/merian/vk/shader/shader_compiler_system_glslc.hpp create mode 100644 src/merian/vk/shader/shader_compiler_shaderc_stub.cpp create mode 100644 src/merian/vk/shader/shader_compiler_system_glslangValidator.cpp create mode 100644 src/merian/vk/shader/shader_compiler_system_glslc.cpp create mode 100755 subprojects/subprocess.wrap diff --git a/.clang-tidy b/.clang-tidy index 018e8b1..889961b 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -18,7 +18,8 @@ Checks: > -readability-redundant-string-init, -readability-identifier-length, -readability-avoid-const-params-in-decls, - -readability-uppercase-literal-suffix + -readability-uppercase-literal-suffix, + -bugprone-easily-swappable-parameters CheckOptions: - { key: readability-identifier-naming.NamespaceCase, value: lower_case } - { key: readability-identifier-naming.ClassCase, value: CamelCase } diff --git a/include/merian/io/file_loader.hpp b/include/merian/io/file_loader.hpp index 1b6b0a2..169454f 100644 --- a/include/merian/io/file_loader.hpp +++ b/include/merian/io/file_loader.hpp @@ -1,6 +1,8 @@ #pragma once +#include "merian/utils/string.hpp" #include +#include #include #include #include @@ -14,9 +16,32 @@ class FileLoader { static bool exists(const std::filesystem::path& path, std::filesystem::file_status file_status = std::filesystem::file_status{}); + template static std::vector load_file(const std::filesystem::path& path) { + if (!exists(path)) { + throw std::runtime_error{ + fmt::format("failed to load {} (does not exist)", path.string())}; + } + + // Open the stream to 'lock' the file. + std::ifstream f(path, std::ios::in | std::ios::binary); + const std::size_t size = std::filesystem::file_size(path); + + if (size % sizeof(T)) { + SPDLOG_WARN("loading {} B of data into a vector quantized to {} B", size, sizeof(T)); + } + + std::vector result((size + sizeof(T) - 1) / sizeof(T), {}); + f.read((char*)result.data(), (std::streamsize)size); + + SPDLOG_DEBUG("load {} of data from {}", format_size(size), path.string()); + + return result; + } + static std::string load_file(const std::filesystem::path& path); - static std::optional search_cwd_parents(const std::filesystem::path& path); + static std::optional + search_cwd_parents(const std::filesystem::path& path); public: FileLoader(const std::set& search_paths = {"./"}) diff --git a/include/merian/utils/filesystem.hpp b/include/merian/utils/filesystem.hpp new file mode 100644 index 0000000..85e70e6 --- /dev/null +++ b/include/merian/utils/filesystem.hpp @@ -0,0 +1,39 @@ +#pragma once + +#include +#include + +#ifdef _WIN32 +#include +#endif + +namespace merian { + +inline std::string temporary_file() { +#ifdef _WIN32 + return std::tmpnam(nullptr); +#else + // std::tmpnam is deprecated + const char* tem_dir_name = std::getenv("TMPDIR"); + std::string tmp_file_name_template = (tem_dir_name != nullptr) ? tem_dir_name : ""; +#ifdef P_tmpdir + if (tmp_file_name_template.empty()) { + tmp_file_name_template = P_tmpdir; + } +#endif + if (tmp_file_name_template.empty()) { + tmp_file_name_template = "/tmp"; + } + + tmp_file_name_template += "/merianXXXXXX"; + const int fd = mkstemp(const_cast(tmp_file_name_template.c_str())); + if (fd > 0) { + // immediately close again... + close(fd); + } + + return tmp_file_name_template; +#endif +} + +} // namespace merian diff --git a/include/merian/vk/context.hpp b/include/merian/vk/context.hpp index e7e5a51..8c385d1 100644 --- a/include/merian/vk/context.hpp +++ b/include/merian/vk/context.hpp @@ -2,6 +2,8 @@ #include "merian/io/file_loader.hpp" #include "merian/utils/concurrent/thread_pool.hpp" +#include +#include #include #include @@ -125,6 +127,7 @@ class Context : public std::enable_shared_from_this { std::string filter_device_name); void find_queues(); void create_device_and_queues(uint32_t preffered_number_compute_queues); + void prepare_shader_include_defines(); private: // Helper void extensions_check_instance_layer_support(); @@ -179,12 +182,14 @@ class Context : public std::enable_shared_from_this { bool instance_extension_enabled(const std::string& name) const; - auto get_context_extensions() const; - const std::vector& get_enabled_device_extensions() const; const std::vector& get_enabled_instance_extensions() const; + const std::vector& get_default_shader_include_paths() const; + + const std::map& get_default_shader_macro_definitions() const; + private: std::unordered_map> extensions; @@ -257,6 +262,10 @@ class Context : public std::enable_shared_from_this { std::weak_ptr cmd_pool_T; // Convenience command pool for compute (can be nullptr in very rare occasions) std::weak_ptr cmd_pool_C; + + + std::vector default_shader_include_paths; + std::map default_shader_macro_definitions; }; } // namespace merian diff --git a/include/merian/vk/extension/extension.hpp b/include/merian/vk/extension/extension.hpp index 8f9552a..e2bc8db 100644 --- a/include/merian/vk/extension/extension.hpp +++ b/include/merian/vk/extension/extension.hpp @@ -3,6 +3,7 @@ #include "merian/vk/context.hpp" #include #include +#include namespace merian { @@ -114,7 +115,7 @@ class Extension { // return strings that should be defined when compiling shaders with Merians shader compiler. // Note that device and instance extensions are automatically defined as // MERIAN_DEVICE_EXT_ENABLE_ and MERIAN_INSTANCE_EXT_ENABLE_ - virtual std::vector shader_defines() { + virtual std::map shader_macro_definitions() { return {}; } diff --git a/include/merian/vk/shader/shader_compiler.hpp b/include/merian/vk/shader/shader_compiler.hpp index e3a3deb..aff286c 100644 --- a/include/merian/vk/shader/shader_compiler.hpp +++ b/include/merian/vk/shader/shader_compiler.hpp @@ -14,6 +14,9 @@ namespace merian { // A compiler for shaders. // // Include paths for the merian-nodes library and context extensions must be automatically added. +class ShaderCompiler; +using ShaderCompilerHandle = std::shared_ptr; + class ShaderCompiler { public: class compilation_failed : public std::runtime_error { @@ -21,27 +24,74 @@ class ShaderCompiler { compilation_failed(const std::string& what) : std::runtime_error(what) {} }; + inline static const std::map EXTENSION_SHADER_STAGE_MAP = + { + {".vert", vk::ShaderStageFlagBits::eVertex}, + {".tesc", vk::ShaderStageFlagBits::eTessellationControl}, + {".tese", vk::ShaderStageFlagBits::eTessellationEvaluation}, + {".geom", vk::ShaderStageFlagBits::eGeometry}, + {".frag", vk::ShaderStageFlagBits::eFragment}, + {".comp", vk::ShaderStageFlagBits::eCompute}, + {".mesh", vk::ShaderStageFlagBits::eMeshEXT}, + {".task", vk::ShaderStageFlagBits::eTaskEXT}, + {".rgen", vk::ShaderStageFlagBits::eRaygenKHR}, + {".rint", vk::ShaderStageFlagBits::eIntersectionKHR}, + {".rahit", vk::ShaderStageFlagBits::eAnyHitKHR}, + {".rchit", vk::ShaderStageFlagBits::eClosestHitKHR}, + {".rmiss", vk::ShaderStageFlagBits::eMissKHR}, + {".rcall", vk::ShaderStageFlagBits::eCallableKHR}, + }; + + inline static const std::map SHADER_STAGE_EXTENSION_MAP = + { + {vk::ShaderStageFlagBits::eVertex, ".vert"}, + {vk::ShaderStageFlagBits::eTessellationControl, ".tesc"}, + {vk::ShaderStageFlagBits::eTessellationEvaluation, ".tese"}, + {vk::ShaderStageFlagBits::eGeometry, ".geom"}, + {vk::ShaderStageFlagBits::eFragment, ".frag"}, + {vk::ShaderStageFlagBits::eCompute, ".comp"}, + {vk::ShaderStageFlagBits::eMeshEXT, ".mesh"}, + {vk::ShaderStageFlagBits::eTaskEXT, ".task"}, + {vk::ShaderStageFlagBits::eRaygenKHR, ".rgen"}, + {vk::ShaderStageFlagBits::eIntersectionKHR, ".rint"}, + {vk::ShaderStageFlagBits::eAnyHitKHR, ".rahit"}, + {vk::ShaderStageFlagBits::eClosestHitKHR, ".rchit"}, + {vk::ShaderStageFlagBits::eMissKHR, ".rmiss"}, + {vk::ShaderStageFlagBits::eCallableKHR, ".rcall"}, + }; + public: - ShaderCompiler(const std::vector& user_include_paths = {}, - const std::map& user_macro_definitions = {}) - : include_paths(user_include_paths), macro_definitions(user_macro_definitions) { - // search merian-shaders - - // add macro definitions from context extensions and enabled instance and device extensions. - } + // Returns any of the available shader compilers. Returns nullptr if none is available. + static ShaderCompilerHandle + get(const ContextHandle& context, + const std::vector& user_include_paths = {}, + const std::map& user_macro_definitions = {}); + + ShaderCompiler(const ContextHandle& context, + const std::vector& user_include_paths = {}, + const std::map& user_macro_definitions = {}); virtual ~ShaderCompiler() = 0; + // ------------------------------------------------ + // Attempt to guess the shader_kind from the file extension if shader_kind = std::nullopt. // // May throw compilation_failed. - std::vector + virtual std::vector compile_glsl(const std::filesystem::path& path, const std::optional optional_shader_kind = std::nullopt) { return compile_glsl(FileLoader::load_file(path), path.string(), optional_shader_kind.value_or(guess_kind(path))); } + // May throw compilation_failed. + virtual std::vector compile_glsl(const std::string& source, + const std::string& source_name, + const vk::ShaderStageFlagBits shader_kind) = 0; + + // ------------------------------------------------ + ShaderModuleHandle compile_glsl_to_shadermodule( const ContextHandle& context, const std::filesystem::path& path, @@ -59,10 +109,7 @@ class ShaderCompiler { context, compile_glsl(source, source_name, shader_kind), shader_kind); } - // May throw compilation_failed. - virtual std::vector compile_glsl(const std::string& source, - const std::string& source_name, - const vk::ShaderStageFlagBits shader_kind) = 0; + // ------------------------------------------------ const std::vector& get_include_paths() const { return include_paths; @@ -72,6 +119,8 @@ class ShaderCompiler { return macro_definitions; } + virtual bool available() const = 0; + private: static vk::ShaderStageFlagBits guess_kind(const std::filesystem::path& path) { std::string extension; @@ -83,23 +132,8 @@ class ShaderCompiler { extension = path.extension().string(); } - if (extension == ".vert") { - return vk::ShaderStageFlagBits::eVertex; - } - if (extension == ".tesc") { - return vk::ShaderStageFlagBits::eTessellationControl; - } - if (extension == ".tese") { - return vk::ShaderStageFlagBits::eTessellationEvaluation; - } - if (extension == ".geom") { - return vk::ShaderStageFlagBits::eGeometry; - } - if (extension == ".frag") { - return vk::ShaderStageFlagBits::eFragment; - } - if (extension == ".comp") { - return vk::ShaderStageFlagBits::eCompute; + if (EXTENSION_SHADER_STAGE_MAP.contains(extension)) { + return EXTENSION_SHADER_STAGE_MAP.at(extension); } throw compilation_failed{ @@ -109,6 +143,5 @@ class ShaderCompiler { std::vector include_paths; std::map macro_definitions; }; -using ShaderCompilerHandle = std::shared_ptr; } // namespace merian diff --git a/include/merian/vk/shader/shader_compiler_shaderc.hpp b/include/merian/vk/shader/shader_compiler_shaderc.hpp index 007707b..6e5b3b5 100644 --- a/include/merian/vk/shader/shader_compiler_shaderc.hpp +++ b/include/merian/vk/shader/shader_compiler_shaderc.hpp @@ -3,13 +3,7 @@ #include "merian/vk/shader/shader_compiler.hpp" #include -#ifdef __has_include -#if !__has_include() -static_assert(false, "shaderc is required for ShadercCompiler"); -#else -#include -#endif -#else +#ifdef MERIAN_SHADERC_FOUND #include #endif @@ -17,7 +11,8 @@ namespace merian { class ShadercCompiler : public ShaderCompiler { public: - ShadercCompiler(const std::vector& include_paths = {}, + ShadercCompiler(const ContextHandle& context, + const std::vector& include_paths = {}, const std::map& macro_definitions = {}); ~ShadercCompiler(); @@ -26,9 +21,13 @@ class ShadercCompiler : public ShaderCompiler { const std::string& source_name, const vk::ShaderStageFlagBits shader_kind) override; + bool available() const override; + private: +#ifdef MERIAN_SHADERC_FOUND shaderc::Compiler shader_compiler; shaderc::CompileOptions compile_options; +#endif }; } // namespace merian diff --git a/include/merian/vk/shader/shader_compiler_glsllangValidator.hpp b/include/merian/vk/shader/shader_compiler_system_glslangValidator.hpp similarity index 55% rename from include/merian/vk/shader/shader_compiler_glsllangValidator.hpp rename to include/merian/vk/shader/shader_compiler_system_glslangValidator.hpp index cfa0025..903e856 100644 --- a/include/merian/vk/shader/shader_compiler_glsllangValidator.hpp +++ b/include/merian/vk/shader/shader_compiler_system_glslangValidator.hpp @@ -6,17 +6,24 @@ namespace merian { // Uses glslangValidator executable to compile shaders. -class GLSLLangValidatorCompiler : public ShaderCompiler { +class SystemGlslangValidatorCompiler : public ShaderCompiler { public: // Include paths for the merian-nodes library are automatically added - GLSLLangValidatorCompiler(const std::vector& include_paths = {}, - const std::map& macro_definitions = {}); + SystemGlslangValidatorCompiler( + const ContextHandle& context, + const std::vector& include_paths = {}, + const std::map& macro_definitions = {}); - ~GLSLLangValidatorCompiler(); + ~SystemGlslangValidatorCompiler(); std::vector compile_glsl(const std::string& source, const std::string& source_name, const vk::ShaderStageFlagBits shader_kind) override; + + bool available() const override; + + private: + const ContextHandle context; }; } // namespace merian diff --git a/include/merian/vk/shader/shader_compiler_system_glslc.hpp b/include/merian/vk/shader/shader_compiler_system_glslc.hpp new file mode 100644 index 0000000..39c1354 --- /dev/null +++ b/include/merian/vk/shader/shader_compiler_system_glslc.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include "merian/vk/shader/shader_compiler.hpp" +#include + +namespace merian { + +// Uses shaderc executable to compile shaders. +class SystemGlslcCompiler : public ShaderCompiler { + public: + // Include paths for the merian-nodes library are automatically added + SystemGlslcCompiler(const ContextHandle& context, + const std::vector& include_paths = {}, + const std::map& macro_definitions = {}); + + ~SystemGlslcCompiler(); + + std::vector compile_glsl(const std::string& source, + const std::string& source_name, + const vk::ShaderStageFlagBits shader_kind) override; + + bool available() const override; + + private: + const ContextHandle context; +}; + +} // namespace merian diff --git a/include/merian/vk/shader/shader_module.hpp b/include/merian/vk/shader/shader_module.hpp index de130f4..766af92 100644 --- a/include/merian/vk/shader/shader_module.hpp +++ b/include/merian/vk/shader/shader_module.hpp @@ -23,7 +23,7 @@ class ShaderModule : public std::enable_shared_from_this { ShaderModule() = delete; ShaderModule(const ContextHandle& context, - const std::string spv_filename, + const std::string& spv_filename, const vk::ShaderStageFlagBits stage_flags = vk::ShaderStageFlagBits::eCompute, const std::optional& file_loader = std::nullopt); diff --git a/meson.build b/meson.build index 78b8dec..17fc59f 100644 --- a/meson.build +++ b/meson.build @@ -43,6 +43,11 @@ add_project_arguments('-DMERIAN_VERSION_MINOR=@0@'.format(version_split[1]), lan add_project_arguments('-DMERIAN_VERSION_PATCH=@0@'.format(version_split[2]), language : 'cpp') add_project_arguments('-DMERIAN_PROJECT_NAME="@0@"'.format(meson.project_name()), language : 'cpp') +add_project_arguments('-DMERIAN_INSTALL_PREFIX="@0@"'.format(get_option('prefix')), language : 'cpp') +add_project_arguments('-DMERIAN_INSTALL_INCLUDE_DIR="@0@"'.format(join_paths(get_option('prefix'), get_option('includedir'))), language : 'cpp') +add_project_arguments('-DMERIAN_INSTALL_DATA_DIR="@0@"'.format(join_paths(get_option('prefix'), get_option('datadir'), meson.project_name())), language : 'cpp') +add_project_arguments('-DMERIAN_DEVELOPMENT_INCLUDE_DIR="@0@"'.format(join_paths(meson.current_source_dir(), 'include')), language : 'cpp') + # Dependencies fmt = dependency('fmt', version : ['>=10.0.0'], fallback : ['fmt', 'fmt_dep']) spdlog = dependency('spdlog', version : ['>=1.12.0'], fallback : ['spdlog', 'spdlog_dep'], default_options : ['external_fmt=enabled']) @@ -52,6 +57,7 @@ if not vulkan.found() vulkan = dependency('vulkan') endif glfw = dependency('glfw3', version: ['>=3.4.0'], fallback: ['glfw', 'glfw_dep']) +subprocess = dependency('subprocess', version: ['>=0.4.0'], fallback: ['subprocess', 'subprocess_dep']) glm = dependency('glm', fallback : ['glm', 'glm_dep'], version: ['>=0.9.9.8']) tol = dependency('tinyobjloader', fallback : ['tinyobjloader', 'tinyobjloader_dep']) tgltf = dependency('tinygltf', required: get_option('tinygltf'), fallback: ['tinygltf', 'tinygltf_dep']) @@ -93,10 +99,6 @@ if not shaderc.found() and get_option('shaderc').enabled() ) endif -if shaderc.found() - add_project_arguments('-DMERIAN_ENABLE_SHADERC', language: 'cpp') -endif - project_root = meson.current_source_dir() inc_dirs = [ include_directories('./include'), @@ -111,6 +113,9 @@ else glslang_options = cmake.subproject_options() glslang_options.add_cmake_defines({'GLSLANG_TESTS': false}) glslang_options.add_cmake_defines({'GLSLANG_ENABLE_INSTALL': false}) + glslang_options.set_install(false) + # installs as glslang_standalone + # glslang_options.set_install(true, target: 'glslang-standalone') glslang_subp = cmake.subproject('glslang', options: glslang_options) glslc_tgt = glslang_subp.target('glslang-standalone') glslc_path = glslc_tgt.full_path() diff --git a/src/merian-nodes/nodes/shadertoy/shadertoy.cpp b/src/merian-nodes/nodes/shadertoy/shadertoy.cpp index f34fc02..edbecf8 100644 --- a/src/merian-nodes/nodes/shadertoy/shadertoy.cpp +++ b/src/merian-nodes/nodes/shadertoy/shadertoy.cpp @@ -3,10 +3,7 @@ #include "merian-nodes/connectors/managed_vk_image_out.hpp" #include "merian-nodes/graph/errors.hpp" #include "merian/vk/pipeline/specialization_info_builder.hpp" - -#if MERIAN_ENABLE_SHADERC #include "merian/vk/shader/shader_compiler_shaderc.hpp" -#endif namespace merian_nodes { @@ -63,19 +60,24 @@ void mainImage(out vec4 fragColor, in vec2 fragCoord) { class ShadertoyInjectCompiler : public ShaderCompiler { public: - ShadertoyInjectCompiler(const ShaderCompilerHandle& forwarding_compiler) - : forwarding_compiler(forwarding_compiler) {} + ShadertoyInjectCompiler(const ContextHandle& context, + const ShaderCompilerHandle& forwarding_compiler) + : ShaderCompiler(context), forwarding_compiler(forwarding_compiler) {} ~ShadertoyInjectCompiler() {} std::vector compile_glsl(const std::string& source, const std::string& source_name, - const vk::ShaderStageFlagBits shader_kind) final { + const vk::ShaderStageFlagBits shader_kind) final override { SPDLOG_INFO("(re-)compiling {}", source_name); return forwarding_compiler->compile_glsl(shadertoy_pre + source + shadertoy_post, source_name, shader_kind); } + bool available() const override { + return true; + } + private: const ShaderCompilerHandle forwarding_compiler; }; @@ -83,16 +85,13 @@ class ShadertoyInjectCompiler : public ShaderCompiler { Shadertoy::Shadertoy(const ContextHandle& context) : AbstractCompute(context, sizeof(PushConstant)), shader_glsl(default_shader) { - ShaderCompilerHandle shaderc_compiler = nullptr; -#if MERIAN_ENABLE_SHADERC - shaderc_compiler = std::make_shared(); -#endif + ShaderCompilerHandle forwarding_compiler = ShaderCompiler::get(context); - if (!shaderc_compiler) { + if (!forwarding_compiler->available()) { return; } - compiler = std::make_shared(shaderc_compiler); + compiler = std::make_shared(context, forwarding_compiler); reloader = std::make_unique(context, compiler); shader = compiler->compile_glsl_to_shadermodule(context, shader_glsl, "Shadertoy.comp", vk::ShaderStageFlagBits::eCompute); diff --git a/src/merian/io/file_loader.cpp b/src/merian/io/file_loader.cpp index fdcc877..b4046fe 100644 --- a/src/merian/io/file_loader.cpp +++ b/src/merian/io/file_loader.cpp @@ -22,10 +22,10 @@ std::string FileLoader::load_file(const std::filesystem::path& path) { // Open the stream to 'lock' the file. std::ifstream f(path, std::ios::in | std::ios::binary); - const auto size = std::filesystem::file_size(path); + const std::size_t size = std::filesystem::file_size(path); std::string result(size, '\0'); - f.read(result.data(), size); + f.read(result.data(), (std::streamsize)size); SPDLOG_DEBUG("load {} of data from {}", format_size(size), path.string()); diff --git a/src/merian/meson.build b/src/merian/meson.build index ea2a5ce..6561f04 100644 --- a/src/merian/meson.build +++ b/src/merian/meson.build @@ -48,6 +48,9 @@ merian_src = files( if shaderc.found() merian_src += files('vk/shader/shader_compiler_shaderc.cpp') + add_project_arguments('-DMERIAN_SHADERC_FOUND', language: 'cpp') +else + merian_src += files('vk/shader/shader_compiler_shaderc_stub.cpp') endif subdir('vk/shader') @@ -64,6 +67,7 @@ merian_lib = static_library( sdl2, shaderc, spdlog, + subprocess, tgltf, tol, vma, diff --git a/src/merian/vk/context.cpp b/src/merian/vk/context.cpp index b2e6000..a9947b0 100644 --- a/src/merian/vk/context.cpp +++ b/src/merian/vk/context.cpp @@ -4,7 +4,6 @@ #include "merian/vk/extension/extension.hpp" #include -#include #include #include @@ -53,11 +52,11 @@ Version: {}\n\n", VK_API_VERSION_MINOR(VK_HEADER_VERSION_COMPLETE), VK_API_VERSION_PATCH(VK_HEADER_VERSION_COMPLETE)); + // add common folders to file loader + file_loader.add_search_path(MERIAN_INSTALL_DATA_DIR); + // Init dynamic loader - static vk::DynamicLoader dl; - static PFN_vkGetInstanceProcAddr vk_get_instance_proc_addr = - dl.getProcAddress("vkGetInstanceProcAddr"); - VULKAN_HPP_DEFAULT_DISPATCHER.init(vk_get_instance_proc_addr); + VULKAN_HPP_DEFAULT_DISPATCHER.init(); SPDLOG_DEBUG("supplied extensions:"); for (const auto& ext : desired_extensions) { @@ -73,6 +72,8 @@ Version: {}\n\n", find_queues(); create_device_and_queues(preffered_number_compute_queues); + prepare_shader_include_defines(); + SPDLOG_INFO("context ready."); } @@ -532,6 +533,48 @@ void Context::create_device_and_queues(uint32_t preferred_number_compute_queues) pipeline_cache = device.createPipelineCache(pipeline_cache_create_info); } +void Context::prepare_shader_include_defines() { + // search merian-shaders + const std::filesystem::path development_headers = + std::filesystem::path(MERIAN_DEVELOPMENT_INCLUDE_DIR); + const std::filesystem::path installed_headers = + std::filesystem::path(MERIAN_INSTALL_INCLUDE_DIR); + + if (FileLoader::exists(development_headers / "merian-shaders")) { + SPDLOG_DEBUG("found merian-shaders development headers headers at {}", development_headers.string()); + default_shader_include_paths.emplace_back(development_headers); + } else if (FileLoader::exists(installed_headers / "merian-shaders")) { + SPDLOG_DEBUG("found merian-shaders installed at {}", installed_headers.string()); + default_shader_include_paths.emplace_back(installed_headers); + } else { + SPDLOG_ERROR("merian-shaders header not found! Shader compilers might not work correctly"); + } + + // add macro definitions from context extensions and enabled instance and device extensions. + for (const auto& ext : extensions) { + const auto extension_macro_definitions = ext.second->shader_macro_definitions(); + default_shader_macro_definitions.insert(extension_macro_definitions.begin(), + extension_macro_definitions.end()); + } + const std::string instance_ext_define_prefix = "MERIAN_INSTANCE_EXT_ENABLE_"; + for (const auto* ext : instance_extension_names) { + default_shader_macro_definitions.emplace(instance_ext_define_prefix + ext, "1"); + } + const std::string device_ext_define_prefix = "MERIAN_DEVICE_EXT_ENABLE_"; + for (const auto* ext : device_extensions) { + default_shader_macro_definitions.emplace(device_ext_define_prefix + ext, "1"); + } + +#ifndef NDEBUG + std::vector string_defines; + string_defines.reserve(default_shader_macro_definitions.size()); + for (const auto& def : default_shader_macro_definitions) { + string_defines.emplace_back(def.first + "=" + def.second); + } + SPDLOG_DEBUG("shader defines: {}", fmt::join(string_defines, "\n")); +#endif +} + //////////// // HELPERS //////////// @@ -736,10 +779,6 @@ bool Context::instance_extension_enabled(const std::string& name) const { [&](const char* s) { return name == s; }) != instance_extension_names.end(); } -auto Context::get_context_extensions() const { - return std::views::values(extensions); -} - const std::vector& Context::get_enabled_device_extensions() const { return device_extensions; } @@ -748,4 +787,12 @@ const std::vector& Context::get_enabled_instance_extensions() const return instance_extension_names; } +const std::vector& Context::get_default_shader_include_paths() const { + return default_shader_include_paths; +} + +const std::map& Context::get_default_shader_macro_definitions() const { + return default_shader_macro_definitions; +} + } // namespace merian diff --git a/src/merian/vk/shader/meson.build b/src/merian/vk/shader/meson.build index 5d3977b..940226f 100644 --- a/src/merian/vk/shader/meson.build +++ b/src/merian/vk/shader/meson.build @@ -2,4 +2,6 @@ merian_src += shader_generator.process('fullscreen_triangle.vert') merian_src += files( 'shader_module.cpp', + 'shader_compiler_system_glslangValidator.cpp', + 'shader_compiler_system_glslc.cpp' ) diff --git a/src/merian/vk/shader/shader_compiler.cpp b/src/merian/vk/shader/shader_compiler.cpp index 5d251f8..61df7ea 100644 --- a/src/merian/vk/shader/shader_compiler.cpp +++ b/src/merian/vk/shader/shader_compiler.cpp @@ -1,7 +1,50 @@ #include "merian/vk/shader/shader_compiler.hpp" +#include "merian/utils/vector.hpp" + +#include "merian/vk/shader/shader_compiler_shaderc.hpp" +#include "merian/vk/shader/shader_compiler_system_glslangValidator.hpp" +#include "merian/vk/shader/shader_compiler_system_glslc.hpp" namespace merian { -ShaderCompiler::~ShaderCompiler(){}; +ShaderCompilerHandle +ShaderCompiler::get(const ContextHandle& context, + const std::vector& user_include_paths, + const std::map& user_macro_definitions) { + ShaderCompilerHandle shaderc = + std::make_shared(context, user_include_paths, user_macro_definitions); + if (shaderc->available()) { + SPDLOG_DEBUG("using shipped shaderc as default compiler"); + return shaderc; + } + + ShaderCompilerHandle glslang_validator = std::make_shared( + context, user_include_paths, user_macro_definitions); + if (glslang_validator->available()) { + SPDLOG_DEBUG("using installed glslangValidator as default compiler"); + return glslang_validator; + } + + ShaderCompilerHandle glslc = + std::make_shared(context, user_include_paths, user_macro_definitions); + if (glslc->available()) { + SPDLOG_DEBUG("using installed glslc as default compiler"); + return glslc; + } + + return nullptr; +} + +ShaderCompiler::ShaderCompiler(const ContextHandle& context, + const std::vector& user_include_paths, + const std::map& user_macro_definitions) + : include_paths(user_include_paths), macro_definitions(user_macro_definitions) { + insert_all(include_paths, context->get_default_shader_include_paths()); + macro_definitions.insert(context->get_default_shader_macro_definitions().begin(), + context->get_default_shader_macro_definitions().end()); } + +ShaderCompiler::~ShaderCompiler(){}; + +} // namespace merian diff --git a/src/merian/vk/shader/shader_compiler_shaderc.cpp b/src/merian/vk/shader/shader_compiler_shaderc.cpp index 9e8330c..a0f6e19 100644 --- a/src/merian/vk/shader/shader_compiler_shaderc.cpp +++ b/src/merian/vk/shader/shader_compiler_shaderc.cpp @@ -104,9 +104,10 @@ shaderc_shader_kind_for_stage_flag_bit(const vk::ShaderStageFlagBits shader_kind } } -ShadercCompiler::ShadercCompiler(const std::vector& include_paths, - const std::map& macro_definitions) - : ShaderCompiler(include_paths, macro_definitions) { +ShadercCompiler::ShadercCompiler(const ContextHandle& context, + const std::vector& user_include_paths, + const std::map& user_macro_definitions) + : ShaderCompiler(context, user_include_paths, user_macro_definitions) { for (const auto& [key, value] : get_macro_definitions()) { compile_options.AddMacroDefinition(key, value); @@ -151,4 +152,8 @@ std::vector ShadercCompiler::compile_glsl(const std::string& source, return std::vector(binary_result.begin(), binary_result.end()); } +bool ShadercCompiler::available() const { + return true; +} + } // namespace merian diff --git a/src/merian/vk/shader/shader_compiler_shaderc_stub.cpp b/src/merian/vk/shader/shader_compiler_shaderc_stub.cpp new file mode 100644 index 0000000..1851a1d --- /dev/null +++ b/src/merian/vk/shader/shader_compiler_shaderc_stub.cpp @@ -0,0 +1,26 @@ +#include "merian/vk/shader/shader_compiler_shaderc.hpp" + +#include + +namespace merian { + +ShadercCompiler::ShadercCompiler(const ContextHandle& context, + const std::vector& include_paths, + const std::map& macro_definitions) + : ShaderCompiler(context, include_paths, macro_definitions) {} + +ShadercCompiler::~ShadercCompiler() {} + +std::vector +ShadercCompiler::compile_glsl([[maybe_unused]] const std::string& source, + [[maybe_unused]] const std::string& source_name, + [[maybe_unused]] const vk::ShaderStageFlagBits shader_kind) { + throw merian::ShaderCompiler::compilation_failed{ + "shaderc is not available (was not found at compile time)"}; +} + +bool ShadercCompiler::available() const { + return false; +} + +} // namespace merian diff --git a/src/merian/vk/shader/shader_compiler_system_glslangValidator.cpp b/src/merian/vk/shader/shader_compiler_system_glslangValidator.cpp new file mode 100644 index 0000000..fc6a30b --- /dev/null +++ b/src/merian/vk/shader/shader_compiler_system_glslangValidator.cpp @@ -0,0 +1,79 @@ +#include "merian/vk/shader/shader_compiler_system_glslangValidator.hpp" +#include "merian/utils/filesystem.hpp" + +#include "subprocess/ProcessBuilder.hpp" +#include "subprocess/basic_types.hpp" +#include "subprocess/shell_utils.hpp" +#include + +namespace merian { + +// Include paths for the merian-nodes library are automatically added +SystemGlslangValidatorCompiler::SystemGlslangValidatorCompiler( + const ContextHandle& context, + const std::vector& user_include_paths, + const std::map& user_macro_definitions) + : ShaderCompiler(context, user_include_paths, user_macro_definitions), context(context) {} + +SystemGlslangValidatorCompiler::~SystemGlslangValidatorCompiler() {} + +std::vector +SystemGlslangValidatorCompiler::compile_glsl(const std::string& source, + [[maybe_unused]] const std::string& source_name, + const vk::ShaderStageFlagBits shader_kind) { + std::vector command = {subprocess::find_program("glslangValidator")}; + + command.emplace_back("--target-env"); + if (context->vk_api_version == VK_API_VERSION_1_0) { + command.emplace_back("vulkan1.0"); + } else if (context->vk_api_version == VK_API_VERSION_1_1) { + command.emplace_back("vulkan1.1"); + } else { + command.emplace_back("vulkan1.3"); + } + + command.emplace_back("--stdin"); + + if (!SHADER_STAGE_EXTENSION_MAP.contains(shader_kind)) { + throw compilation_failed{ + fmt::format("shader kind {} unsupported.", vk::to_string(shader_kind))}; + } + + command.emplace_back("-S"); + command.emplace_back(SHADER_STAGE_EXTENSION_MAP.at(shader_kind).substr(1)); + + for (const auto& inc_dir : get_include_paths()) { + command.emplace_back(fmt::format("-I{}", inc_dir)); + } + for (const auto& macro_def : get_macro_definitions()) { + command.emplace_back(fmt::format("-D{}={}", macro_def.first, macro_def.second)); + } + + const std::string output_file = temporary_file(); + command.emplace_back("-o"); + command.emplace_back(output_file); + + SPDLOG_DEBUG("running command {}", fmt::join(command, " ")); + subprocess::CompletedProcess process = + subprocess::run(command, subprocess::RunBuilder() + .cin(source) + .cerr(subprocess::PipeOption::pipe) + .cout(subprocess::PipeOption::pipe)); + + if (process.returncode != 0) { + throw compilation_failed{fmt::format("glslangValidator command failed:\n{}\n\n{}\n\n{}", + process.cout, process.cerr, fmt::join(command, " "))}; + } + + std::vector spv = FileLoader::load_file(output_file); + + std::filesystem::remove(output_file); + + return spv; +} + +bool SystemGlslangValidatorCompiler::available() const { + return !subprocess::find_program("glslangValidator").empty(); +} + +} // namespace merian diff --git a/src/merian/vk/shader/shader_compiler_system_glslc.cpp b/src/merian/vk/shader/shader_compiler_system_glslc.cpp new file mode 100644 index 0000000..700e5cd --- /dev/null +++ b/src/merian/vk/shader/shader_compiler_system_glslc.cpp @@ -0,0 +1,80 @@ +#include "merian/vk/shader/shader_compiler_system_glslc.hpp" + +#include "subprocess/ProcessBuilder.hpp" +#include "subprocess/basic_types.hpp" +#include "subprocess/shell_utils.hpp" + +#include + +namespace merian { + +// Include paths for the merian-nodes library are automatically added +SystemGlslcCompiler::SystemGlslcCompiler( + const ContextHandle& context, + const std::vector& user_include_paths, + const std::map& user_macro_definitions) + : ShaderCompiler(context, user_include_paths, user_macro_definitions), context(context) {} + +SystemGlslcCompiler::~SystemGlslcCompiler() {} + + + +std::vector +SystemGlslcCompiler::compile_glsl(const std::string& source, + [[maybe_unused]] const std::string& source_name, + const vk::ShaderStageFlagBits shader_kind) { + std::vector command = {subprocess::find_program("glslc")}; + + if (context->vk_api_version == VK_API_VERSION_1_0) { + command.emplace_back("--target-env=vulkan1.0"); + } else if (context->vk_api_version == VK_API_VERSION_1_1) { + command.emplace_back("--target-env=vulkan1.1"); + } else if (context->vk_api_version == VK_API_VERSION_1_2) { + command.emplace_back("--target-env=vulkan1.2"); + } else { + command.emplace_back("--target-env=vulkan1.3"); + } + + if (!SHADER_STAGE_EXTENSION_MAP.contains(shader_kind)) { + throw compilation_failed{ + fmt::format("shader kind {} unsupported.", vk::to_string(shader_kind))}; + } + + command.emplace_back(fmt::format("-fshader-stage={}", SHADER_STAGE_EXTENSION_MAP.at(shader_kind).substr(1))); + + for (const auto& inc_dir : get_include_paths()) { + command.emplace_back("-I"); + command.emplace_back(inc_dir); + } + for (const auto& macro_def : get_macro_definitions()) { + command.emplace_back(fmt::format("-D{}={}", macro_def.first, macro_def.second)); + } + + command.emplace_back("-"); + + command.emplace_back("-o"); + command.emplace_back("-"); + + SPDLOG_DEBUG("running command {}", fmt::join(command, " ")); + subprocess::CompletedProcess process = + subprocess::run(command, subprocess::RunBuilder() + .cin(source) + .cerr(subprocess::PipeOption::pipe) + .cout(subprocess::PipeOption::pipe)); + + if (process.returncode != 0) { + throw compilation_failed{fmt::format("glslc command failed:\n{}\n\n{}\n\n{}", + process.cout, process.cerr, fmt::join(command, " "))}; + } + + std::vector spv((process.cout.size() + sizeof(uint32_t) - 1) / sizeof(uint32_t)); + memcpy(spv.data(), process.cout.data(), process.cout.size()); + + return spv; +} + +bool SystemGlslcCompiler::available() const { + return !subprocess::find_program("glslc").empty(); +} + +} // namespace merian diff --git a/src/merian/vk/shader/shader_module.cpp b/src/merian/vk/shader/shader_module.cpp index af4e8c4..f3b4446 100644 --- a/src/merian/vk/shader/shader_module.cpp +++ b/src/merian/vk/shader/shader_module.cpp @@ -4,7 +4,7 @@ namespace merian { ShaderModule::ShaderModule(const ContextHandle& context, - const std::string spv_filename, + const std::string& spv_filename, const vk::ShaderStageFlagBits stage_flags, const std::optional& file_loader) : context(context), stage_flags(stage_flags) { diff --git a/subprojects/subprocess.wrap b/subprojects/subprocess.wrap new file mode 100755 index 0000000..e7b71d7 --- /dev/null +++ b/subprojects/subprocess.wrap @@ -0,0 +1,11 @@ +[wrap-git] +directory = subprocess + +url = https://github.com/benman64/subprocess +revision = 1b35a489e92ebe261ce781a65d7bf47d71be3655 +depth = 1 +method = cmake +clone-recursive = true + +[provide] +subprocess = subprocess_dep