Gui Stencil Mask Optimization

assets/test/shaders/demo_8_gui_stencil.frag.glsl

@@ -0,0 +1,20 @@
+#version 330
+
+in vec2 v_uv;
+uniform float time; // Add time uniform for animation
+
+out vec4 fragColor;
+
+void main() {
+	// Create moving gradient based on UV coordinates and time
+	vec2 moving_uv = v_uv + vec2(sin(time * 0.5) * 0.5, cos(time * 0.3) * 0.5);
+
+	// Create some colorful pattern
+	vec3 color1 = vec3(0.2, 0.5, 0.8); // Blue-ish
+	vec3 color2 = vec3(0.8, 0.2, 0.5); // Pink-ish
+
+	vec3 color = mix(color1, color2, sin(moving_uv.x * 5.0 + time) * 0.5 + 0.5);
+	color += vec3(0.2) * sin(moving_uv.y * 10.0 + time * 2.0);
+
+	fragColor = vec4(color, 1.0);
+}

assets/test/shaders/demo_8_gui_stencil.vert.glsl

@@ -0,0 +1,11 @@
+#version 330
+
+layout(location = 0) in vec2 position;
+layout(location = 1) in vec2 uv;
+
+out vec2 v_uv;
+
+void main() {
+	gl_Position = vec4(position, 0.0, 1.0);
+	v_uv = uv;
+}

assets/test/shaders/demo_8_simple_gui.frag.glsl

@@ -0,0 +1,17 @@
+#version 330
+
+in vec2 v_uv;
+out vec4 fragColor;
+
+void main() {
+	// Semi-transparent white color for the GUI
+	vec4 gui_color = vec4(1.0, 1.0, 1.0, 1.0);
+
+	// Add a border to make it look more like a GUI window
+	float border = 0.05;
+	if (v_uv.x < border || v_uv.x > 1.0 - border || v_uv.y < border || v_uv.y > 1.0 - border) {
+		gui_color = vec4(0.8, 0.8, 0.8, 1.0); // Darker border
+	}
+
+	fragColor = gui_color;
+}

assets/test/shaders/demo_8_simple_gui.vert.glsl

@@ -0,0 +1,15 @@
+#version 330
+
+layout(location = 0) in vec2 position;
+layout(location = 1) in vec2 uv;
+
+out vec2 v_uv;
+
+void main() {
+	// Scale down the quad to show part of the background
+	vec2 scaled_pos = position * 0.5; // Make it half size
+	// Move it to top-right corner
+	scaled_pos += vec2(0.5, 0.5);
+	gl_Position = vec4(scaled_pos, 0.0, 1.0);
+	v_uv = uv;
+}

libopenage/renderer/demo/CMakeLists.txt

@@ -6,6 +6,7 @@ add_sources(libopenage
     demo_4.cpp
     demo_5.cpp
     demo_6.cpp
+    demo_8.cpp
     stresstest_0.cpp
     stresstest_1.cpp
 	tests.cpp

libopenage/renderer/demo/demo_8.cpp

@@ -0,0 +1,175 @@
+// Copyright 2025-2025 the openage authors. See copying.md for legal info.
+
+#include "demo_8.h"
+
+#include "renderer/demo/util.h"
+#include "renderer/gui/gui.h"
+#include "renderer/gui/integration/public/gui_application_with_logger.h"
+#include "renderer/opengl/render_pass.h"
+#include "renderer/opengl/renderer.h"
+#include "renderer/opengl/window.h"
+#include "renderer/render_pass.h"
+#include "renderer/render_target.h"
+#include "renderer/resources/mesh_data.h"
+#include "renderer/resources/shader_source.h"
+#include "renderer/shader_program.h"
+
+
+namespace openage::renderer::tests {
+
+void renderer_demo_8(const util::Path &path) {
+	auto qtapp = std::make_shared<gui::GuiApplicationWithLogger>();
+
+	// Create window and renderer
+	window_settings settings;
+	settings.width = 800;
+	settings.height = 600;
+	settings.debug = true;
+	auto window = std::make_shared<opengl::GlWindow>("openage renderer stencil test", settings);
+	auto renderer = window->make_renderer();
+
+	auto shaderdir = path / "assets" / "test" / "shaders";
+
+	// Create background shader (something that would be masked by GUI)
+	auto bg_vshader_file = (shaderdir / "demo_8_gui_stencil.vert.glsl").open();
+	auto bg_vshader_src = resources::ShaderSource(
+		resources::shader_lang_t::glsl,
+		resources::shader_stage_t::vertex,
+		bg_vshader_file.read());
+	bg_vshader_file.close();
+
+	auto bg_fshader_file = (shaderdir / "demo_8_gui_stencil.frag.glsl").open();
+	auto bg_fshader_src = resources::ShaderSource(
+		resources::shader_lang_t::glsl,
+		resources::shader_stage_t::fragment,
+		bg_fshader_file.read());
+	bg_fshader_file.close();
+
+	// Load simple GUI quad shader (using a simple rectangle for GUI)
+	auto gui_vshader_file = (shaderdir / "demo_8_simple_gui.vert.glsl").open();
+	auto gui_vshader_src = resources::ShaderSource(
+		resources::shader_lang_t::glsl,
+		resources::shader_stage_t::vertex,
+		gui_vshader_file.read());
+	gui_vshader_file.close();
+
+	auto gui_fshader_file = (shaderdir / "demo_8_simple_gui.frag.glsl").open();
+	auto gui_fshader_src = resources::ShaderSource(
+		resources::shader_lang_t::glsl,
+		resources::shader_stage_t::fragment,
+		gui_fshader_file.read());
+	gui_fshader_file.close();
+
+	// Load display shader
+	auto display_vshader_file = (shaderdir / "demo_1_display.vert.glsl").open();
+	auto display_vshader_src = resources::ShaderSource(
+		resources::shader_lang_t::glsl,
+		resources::shader_stage_t::vertex,
+		display_vshader_file.read());
+	display_vshader_file.close();
+
+	auto display_fshader_file = (shaderdir / "demo_1_display.frag.glsl").open();
+	auto display_fshader_src = resources::ShaderSource(
+		resources::shader_lang_t::glsl,
+		resources::shader_stage_t::fragment,
+		display_fshader_file.read());
+	display_fshader_file.close();
+
+	auto bg_shader = renderer->add_shader({bg_vshader_src, bg_fshader_src});
+	auto gui_shader = renderer->add_shader({gui_vshader_src, gui_fshader_src});
+	auto display_shader = renderer->add_shader({display_vshader_src, display_fshader_src});
+
+	auto quad = renderer->add_mesh_geometry(resources::MeshData::make_quad());
+
+	auto color_texture = renderer->add_texture(
+		resources::Texture2dInfo(
+			settings.width, settings.height, resources::pixel_format::rgba8));
+
+	auto depth_stencil_texture = renderer->add_texture(
+		resources::Texture2dInfo(
+			settings.width, settings.height, resources::pixel_format::depth24_stencil8));
+	auto fbo = renderer->create_texture_target({color_texture, depth_stencil_texture});
+
+	auto bg_uniforms = bg_shader->new_uniform_input();
+	auto gui_uniforms = gui_shader->new_uniform_input();
+	auto color_texture_uniform = display_shader->new_uniform_input("color_texture", color_texture);
+
+	Renderable bg_obj{
+		bg_uniforms,
+		quad,
+		false,
+		false, // Disable depth test to make stencil testing more visible
+	};
+
+	Renderable gui_obj{
+		gui_uniforms,
+		quad,
+		false,
+		false,
+	};
+
+	Renderable display_obj{
+		color_texture_uniform,
+		quad,
+		false,
+		false,
+	};
+
+	// stencil pass for background and gui
+	auto bg_pass = renderer->add_render_pass({bg_obj}, fbo);
+	auto gui_pass = renderer->add_render_pass({gui_obj}, fbo);
+	// final output pass
+	auto display_pass = renderer->add_render_pass({display_obj}, renderer->get_display_target());
+
+	// Configure stencil configurations
+	auto gl_bg_pass = std::dynamic_pointer_cast<opengl::GlRenderPass>(bg_pass);
+	if (gl_bg_pass) {
+		renderer::opengl::StencilConfig config;
+		config.enabled = true;
+		config.write = false;
+		config.ref_value = 255; // get a white stencil mask in RenderDoc for debugging
+		config.test_func = GL_NOTEQUAL;
+		config.depth_pass = GL_KEEP;
+		gl_bg_pass->set_stencil_config(config);
+	}
+
+	auto gl_gui_stencil_pass = std::dynamic_pointer_cast<opengl::GlRenderPass>(gui_pass);
+	if (gl_gui_stencil_pass) {
+		renderer::opengl::StencilConfig config;
+		config.enabled = true;
+		config.write = true;
+		config.ref_value = 255;
+		config.test_func = GL_ALWAYS;
+		config.depth_pass = GL_REPLACE;
+		gl_gui_stencil_pass->set_stencil_config(config);
+	}
+
+	log::log(INFO << "Stencil Test Demo Instructions:");
+	log::log(INFO << "  1. GUI elements will create a stencil mask");
+	log::log(INFO << "  2. Background will only render in non-GUI areas");
+
+	float current_time = 0.0f;
+
+	while (not window->should_close()) {
+		glClear(GL_STENCIL_BUFFER_BIT);
+
+		current_time += 0.01f;
+		bg_uniforms->update("time", current_time);
+
+		// Render background and GUI
+		renderer->render(gui_pass);
+
+		renderer->render(bg_pass);
+
+		// Render display pass
+		renderer->render(display_pass);
+
+		window->update();
+		qtapp->process_events();
+
+		renderer->check_error();
+	}
+	window->close();
+}
+
+} // namespace openage::renderer::tests

libopenage/renderer/demo/demo_8.h

@@ -0,0 +1,18 @@
+// Copyright 2025-2025 the openage authors. See copying.md for legal info.
+
+#pragma once
+
+#include "util/path.h"
+
+namespace openage::renderer::tests {
+
+/**
+ * Show a demo of stencil testing.
+ *     - GUI elements will create a stencil mask.
+ *     - Background will only render in non-GUI areas.
+ *
+ * @param path Path to the project rootdir.
+ */
+void renderer_demo_8(const util::Path &path);
+
+} // namespace openage::renderer::tests

libopenage/renderer/demo/tests.cpp

@@ -1,4 +1,4 @@
-// Copyright 2015-2024 the openage authors. See copying.md for legal info.
+// Copyright 2015-2025 the openage authors. See copying.md for legal info.
 
 #include "tests.h"
 
@@ -12,6 +12,7 @@
 #include "renderer/demo/demo_4.h"
 #include "renderer/demo/demo_5.h"
 #include "renderer/demo/demo_6.h"
+#include "renderer/demo/demo_8.h"
 #include "renderer/demo/stresstest_0.h"
 #include "renderer/demo/stresstest_1.h"
 
@@ -47,6 +48,10 @@ void renderer_demo(int demo_id, const util::Path &path) {
 		renderer_demo_6(path);
 		break;
 
+	case 8:
+		renderer_demo_8(path);
+		break;
+
 	default:
 		log::log(MSG(err) << "Unknown renderer demo requested: " << demo_id << ".");
 		break;

libopenage/renderer/gui/gui.cpp

@@ -1,4 +1,4 @@
-// Copyright 2015-2024 the openage authors. See copying.md for legal info.
+// Copyright 2015-2025 the openage authors. See copying.md for legal info.
 
 #include "gui.h"
 
@@ -7,6 +7,7 @@
 #include "renderer/gui/guisys/public/gui_renderer.h"
 #include "renderer/gui/integration/public/gui_application_with_logger.h"
 #include "renderer/opengl/context.h"
+#include "renderer/opengl/render_pass.h"
 #include "renderer/render_pass.h"
 #include "renderer/render_target.h"
 #include "renderer/renderer.h"
@@ -85,19 +86,31 @@ void GUI::initialize_render_pass(size_t width,
 	// GUI draw surface. gets drawn on top of the gameworld in the presenter.
 	auto output_texture = this->renderer->add_texture(
 		resources::Texture2dInfo(width, height, resources::pixel_format::rgba8));
-	auto fbo = this->renderer->create_texture_target({output_texture});
+	auto depth_stencil_texture = this->renderer->add_texture(
+		resources::Texture2dInfo(width, height, resources::pixel_format::depth24_stencil8));
+	auto fbo = this->renderer->create_texture_target({output_texture, depth_stencil_texture});
 
 	this->texture_unif = maptex_shader->new_uniform_input("texture", this->texture);
 	Renderable display_obj{
 		this->texture_unif,
 		quad,
 		true,
-		true,
+		false,
 	};
 
 	// TODO: Rendering into the FBO is a bit redundant right now because we
 	// just copy the GUI texture into the output texture
 	this->render_pass = renderer->add_render_pass({display_obj}, fbo);
+
+	auto gl_pass = std::dynamic_pointer_cast<opengl::GlRenderPass>(this->render_pass);
+	if (gl_pass) {
+		renderer::opengl::StencilConfig config;
+		config.enabled = true;
+		config.write = true;
+		config.ref_value = 1;
+		config.test_func = GL_ALWAYS;
+		gl_pass->set_stencil_config(config);
+	}
 }
 
 

libopenage/renderer/opengl/framebuffer.cpp

@@ -1,4 +1,4 @@
-// Copyright 2017-2024 the openage authors. See copying.md for legal info.
+// Copyright 2017-2025 the openage authors. See copying.md for legal info.
 
 #include "framebuffer.h"
 
@@ -42,6 +42,9 @@ GlFramebuffer::GlFramebuffer(const std::shared_ptr<GlContext> &context,
 		if (texture->get_info().get_format() == resources::pixel_format::depth24) {
 			glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_TEXTURE_2D, texture->get_handle(), 0);
 		}
+		else if (texture->get_info().get_format() == resources::pixel_format::depth24_stencil8) {
+			glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, GL_TEXTURE_2D, texture->get_handle(), 0);
+		}
 		else {
 			auto attachmentPoint = GL_COLOR_ATTACHMENT0 + colorTextureCount++;
 			glFramebufferTexture2D(GL_FRAMEBUFFER, attachmentPoint, GL_TEXTURE_2D, texture->get_handle(), 0);

libopenage/renderer/opengl/lookup.h

@@ -1,4 +1,4 @@
-// Copyright 2018-2024 the openage authors. See copying.md for legal info.
+// Copyright 2018-2025 the openage authors. See copying.md for legal info.
 
 // Lookup tables for translating between OpenGL-specific values and generic renderer values,
 // as well as mapping things like type sizes within OpenGL.
@@ -25,7 +25,8 @@ static constexpr auto GL_PIXEL_FORMAT = datastructure::create_const_map<resource
 	std::pair(resources::pixel_format::bgr8, std::tuple(GL_RGB8, GL_BGR, GL_UNSIGNED_BYTE)),
 	std::pair(resources::pixel_format::rgba8, std::tuple(GL_RGBA8, GL_RGBA, GL_UNSIGNED_BYTE)),
 	std::pair(resources::pixel_format::rgba8ui, std::tuple(GL_RGBA8UI, GL_RGBA_INTEGER, GL_UNSIGNED_BYTE)),
-	std::pair(resources::pixel_format::depth24, std::tuple(GL_DEPTH_COMPONENT, GL_DEPTH_COMPONENT, GL_UNSIGNED_BYTE)));
+	std::pair(resources::pixel_format::depth24, std::tuple(GL_DEPTH_COMPONENT, GL_DEPTH_COMPONENT, GL_UNSIGNED_BYTE)),
+	std::pair(resources::pixel_format::depth24_stencil8, std::tuple(GL_DEPTH24_STENCIL8, GL_DEPTH_STENCIL, GL_UNSIGNED_INT_24_8)));
 
 /// Sizes of various uniform/vertex input types in shaders.
 static constexpr auto GL_UNIFORM_TYPE_SIZE = datastructure::create_const_map<GLenum, size_t>(

libopenage/renderer/opengl/render_pass.cpp

@@ -1,4 +1,4 @@
-// Copyright 2019-2024 the openage authors. See copying.md for legal info.
+// Copyright 2019-2025 the openage authors. See copying.md for legal info.
 
 #include "render_pass.h"
 
@@ -35,4 +35,13 @@ bool GlRenderPass::get_is_optimized() const {
 void GlRenderPass::set_is_optimized(bool flag) {
 	this->is_optimized = flag;
 }
+
+void GlRenderPass::set_stencil_config(const StencilConfig config) {
+	this->stencil_config = config;
+}
+
+const StencilConfig &GlRenderPass::get_stencil_config() const {
+	return this->stencil_config;
+}
+
 } // namespace openage::renderer::opengl