Implement support for FSAA in combination with post-processing (minet…

…est#15392)

- Actually it's MSAA I think, or perhaps the terms are equivalent
- I've made it fit into the existing Irrlicht architecture, but that has resulted in code duplication compared to my original "hacky" approach
- OpenGL 3.2+ and OpenGL ES 3.1+ are supported
- EDT_OPENGL3 is not required, EDT_OPENGL works too
- Helpful tutorial: https://learnopengl.com/Advanced-OpenGL/Anti-Aliasing, section "Off-screen MSAA"
- This may be rough around the edges, but in general it works

builtin/settingtypes.txt

@@ -410,10 +410,12 @@ anisotropic_filter (Anisotropic filtering) bool false
 #    * None - No antialiasing (default)
 #
 #    * FSAA - Hardware-provided full-screen antialiasing
-#             (incompatible with Post Processing and Undersampling)
 #             A.K.A multi-sample antialiasing (MSAA)
 #             Smoothens out block edges but does not affect the insides of textures.
-#             A restart is required to change this option.
+#
+#             If Post Processing is disabled, changing FSAA requires a restart.
+#             Also, if Post Processing is disabled, FSAA will not work together with
+#             undersampling or a non-default "3d_mode" setting.
 #
 #    * FXAA - Fast approximate antialiasing
 #             Applies a post-processing filter to detect and smoothen high-contrast edges.

irr/include/EDriverFeatures.h

@@ -129,6 +129,9 @@ enum E_VIDEO_DRIVER_FEATURE
 	//! Support for clamping vertices beyond far-plane to depth instead of capping them.
 	EVDF_DEPTH_CLAMP,
 
+	//! Support for multisample textures.
+	EVDF_TEXTURE_MULTISAMPLE,
+
 	//! Only used for counting the elements of this enum
 	EVDF_COUNT
 };

irr/include/IRenderTarget.h

@@ -26,6 +26,7 @@ enum E_CUBE_SURFACE
 };
 
 //! Interface of a Render Target.
+/** This is a framebuffer object (FBO) in OpenGL. */
 class IRenderTarget : public virtual IReferenceCounted
 {
 public:

irr/include/ITexture.h

@@ -156,6 +156,9 @@ enum E_TEXTURE_TYPE
 	//! 2D texture.
 	ETT_2D,
 
+	//! 2D texture with multisampling.
+	ETT_2D_MS,
+
 	//! Cubemap texture.
 	ETT_CUBEMAP
 };

irr/include/IVideoDriver.h

@@ -273,6 +273,14 @@ class IVideoDriver : public virtual IReferenceCounted
 	virtual ITexture *addRenderTargetTexture(const core::dimension2d<u32> &size,
 			const io::path &name = "rt", const ECOLOR_FORMAT format = ECF_UNKNOWN) = 0;
 
+	//! Adds a multisampled render target texture to the texture cache.
+	/** \param msaa The number of samples to use, values that make sense are > 1.
+	Only works if the driver supports the EVDF_TEXTURE_MULTISAMPLE feature,
+	check via queryFeature.
+	\see addRenderTargetTexture */
+	virtual ITexture *addRenderTargetTextureMs(const core::dimension2d<u32> &size, u8 msaa,
+			const io::path &name = "rt", const ECOLOR_FORMAT format = ECF_UNKNOWN) = 0;
+
 	//! Adds a new render target texture with 6 sides for a cubemap map to the texture cache.
 	/** \param sideLen Length of one cubemap side.
 	\param name A name for the texture. Later calls of getTexture() with this name will return this texture.
@@ -358,6 +366,10 @@ class IVideoDriver : public virtual IReferenceCounted
 	//! Remove all render targets.
 	virtual void removeAllRenderTargets() = 0;
 
+	//! Blit contents of one render target to another one.
+	/** This is glBlitFramebuffer in OpenGL. */
+	virtual void blitRenderTarget(IRenderTarget *from, IRenderTarget *to) = 0;
+
 	//! Sets a boolean alpha channel on the texture based on a color key.
 	/** This makes the texture fully transparent at the texels where
 	this color key can be found when using for example draw2DImage

irr/src/CNullDriver.cpp

@@ -1678,6 +1678,12 @@ ITexture *CNullDriver::addRenderTargetTexture(const core::dimension2d<u32> &size
 	return 0;
 }
 
+ITexture *CNullDriver::addRenderTargetTextureMs(const core::dimension2d<u32> &size, u8 msaa,
+		const io::path &name, const ECOLOR_FORMAT format)
+{
+	return 0;
+}
+
 ITexture *CNullDriver::addRenderTargetTextureCubemap(const irr::u32 sideLen,
 		const io::path &name, const ECOLOR_FORMAT format)
 {

irr/src/CNullDriver.h

@@ -227,6 +227,10 @@ class CNullDriver : public IVideoDriver, public IGPUProgrammingServices
 	virtual ITexture *addRenderTargetTexture(const core::dimension2d<u32> &size,
 			const io::path &name, const ECOLOR_FORMAT format = ECF_UNKNOWN) override;
 
+	//! Creates a multisampled render target texture.
+	virtual ITexture *addRenderTargetTextureMs(const core::dimension2d<u32> &size, u8 msaa,
+			const io::path &name, const ECOLOR_FORMAT format = ECF_UNKNOWN) override;
+
 	//! Creates a render target texture for a cubemap
 	ITexture *addRenderTargetTextureCubemap(const irr::u32 sideLen,
 			const io::path &name, const ECOLOR_FORMAT format) override;
@@ -410,6 +414,8 @@ class CNullDriver : public IVideoDriver, public IGPUProgrammingServices
 	//! Create render target.
 	IRenderTarget *addRenderTarget() override;
 
+	void blitRenderTarget(IRenderTarget *from, IRenderTarget *to) override {}
+
 	//! Remove render target.
 	void removeRenderTarget(IRenderTarget *renderTarget) override;
 

irr/src/COpenGLCoreCacheHandler.h

@@ -85,13 +85,19 @@ class COpenGLCoreCacheHandler
 								GL.BindTexture(prevTextureType, 0);
 
 #if defined(IRR_COMPILE_GL_COMMON)
-								GL.Disable(prevTextureType);
-								GL.Enable(curTextureType);
+								// The "enable/disable texture" stuff is so legacy that
+								// it's not even allowed for multisample textures.
+								// (IRR_COMPILE_GL_COMMON is for the legacy driver.)
+								if (prevTextureType != GL_TEXTURE_2D_MULTISAMPLE)
+									GL.Disable(prevTextureType);
+								if (curTextureType != GL_TEXTURE_2D_MULTISAMPLE)
+									GL.Enable(curTextureType);
 #endif
 							}
 #if defined(IRR_COMPILE_GL_COMMON)
 							else if (!prevTexture)
-								GL.Enable(curTextureType);
+								if (curTextureType != GL_TEXTURE_2D_MULTISAMPLE)
+									GL.Enable(curTextureType);
 #endif
 
 							GL.BindTexture(curTextureType, static_cast<const TOpenGLTexture *>(texture)->getOpenGLTextureName());
@@ -110,7 +116,8 @@ class COpenGLCoreCacheHandler
 						GL.BindTexture(prevTextureType, 0);
 
 #if defined(IRR_COMPILE_GL_COMMON)
-						GL.Disable(prevTextureType);
+						if (prevTextureType != GL_TEXTURE_2D_MULTISAMPLE)
+							GL.Disable(prevTextureType);
 #endif
 					}
 

irr/src/COpenGLCoreRenderTarget.h

@@ -5,6 +5,7 @@
 #pragma once
 
 #include "IRenderTarget.h"
+#include <stdexcept>
 
 #ifndef GL_FRAMEBUFFER_INCOMPLETE_FORMATS
 #define GL_FRAMEBUFFER_INCOMPLETE_FORMATS GL_FRAMEBUFFER_INCOMPLETE_FORMATS_EXT
@@ -122,7 +123,7 @@ class COpenGLCoreRenderTarget : public IRenderTarget
 			TOpenGLTexture *currentTexture = (depthStencil && depthStencil->getDriverType() == DriverType) ? static_cast<TOpenGLTexture *>(depthStencil) : 0;
 
 			if (currentTexture) {
-				if (currentTexture->getType() == ETT_2D) {
+				if (currentTexture->getType() == ETT_2D || currentTexture->getType() == ETT_2D_MS) {
 					GLuint textureID = currentTexture->getOpenGLTextureName();
 
 					const ECOLOR_FORMAT textureFormat = (textureID != 0) ? depthStencil->getColorFormat() : ECF_UNKNOWN;
@@ -172,7 +173,20 @@ class COpenGLCoreRenderTarget : public IRenderTarget
 
 					if (textureID != 0) {
 						AssignedTextures[i] = GL_COLOR_ATTACHMENT0 + i;
-						GLenum textarget = currentTexture->getType() == ETT_2D ? GL_TEXTURE_2D : GL_TEXTURE_CUBE_MAP_POSITIVE_X + (int)CubeSurfaces[i];
+						GLenum textarget;
+						switch (currentTexture->getType()) {
+						case ETT_2D:
+							textarget = GL_TEXTURE_2D;
+							break;
+						case ETT_2D_MS:
+							textarget = GL_TEXTURE_2D_MULTISAMPLE;
+							break;
+						case ETT_CUBEMAP:
+							textarget = GL_TEXTURE_CUBE_MAP_POSITIVE_X + (int)CubeSurfaces[i];
+							break;
+						default:
+							throw std::logic_error("not reachable");
+						}
 						Driver->irrGlFramebufferTexture2D(GL_FRAMEBUFFER, AssignedTextures[i], textarget, textureID, 0);
 						TEST_GL_ERROR(Driver);
 					} else if (AssignedTextures[i] != GL_NONE) {
@@ -198,36 +212,50 @@ class COpenGLCoreRenderTarget : public IRenderTarget
 			// Set depth and stencil attachments.
 
 			if (RequestDepthStencilUpdate) {
-				const ECOLOR_FORMAT textureFormat = (DepthStencil) ? DepthStencil->getColorFormat() : ECF_UNKNOWN;
+				const ECOLOR_FORMAT textureFormat = DepthStencil ? DepthStencil->getColorFormat() : ECF_UNKNOWN;
 
 				if (IImage::isDepthFormat(textureFormat)) {
+					GLenum textarget;
+					switch (DepthStencil->getType()) {
+					case ETT_2D:
+						textarget = GL_TEXTURE_2D;
+						break;
+					case ETT_2D_MS:
+						textarget = GL_TEXTURE_2D_MULTISAMPLE;
+						break;
+					default:
+						// ETT_CUBEMAP is rejected for depth/stencil by setTextures
+						throw std::logic_error("not reachable");
+					}
+
 					GLuint textureID = static_cast<TOpenGLTexture *>(DepthStencil)->getOpenGLTextureName();
 
 #ifdef _IRR_EMSCRIPTEN_PLATFORM_ // The WEBGL_depth_texture extension does not allow attaching stencil+depth separate.
 					if (textureFormat == ECF_D24S8) {
 						GLenum attachment = 0x821A; // GL_DEPTH_STENCIL_ATTACHMENT
-						Driver->irrGlFramebufferTexture2D(GL_FRAMEBUFFER, attachment, GL_TEXTURE_2D, textureID, 0);
+						Driver->irrGlFramebufferTexture2D(GL_FRAMEBUFFER, attachment, textarget, textureID, 0);
 						AssignedStencil = true;
 					} else {
-						Driver->irrGlFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_TEXTURE_2D, textureID, 0);
+						Driver->irrGlFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, textarget, textureID, 0);
 						AssignedStencil = false;
 					}
 #else
-					Driver->irrGlFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_TEXTURE_2D, textureID, 0);
+					Driver->irrGlFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, textarget, textureID, 0);
 
 					if (textureFormat == ECF_D24S8) {
-						Driver->irrGlFramebufferTexture2D(GL_FRAMEBUFFER, GL_STENCIL_ATTACHMENT, GL_TEXTURE_2D, textureID, 0);
+						Driver->irrGlFramebufferTexture2D(GL_FRAMEBUFFER, GL_STENCIL_ATTACHMENT, textarget, textureID, 0);
 
 						AssignedStencil = true;
 					} else {
 						if (AssignedStencil)
-							Driver->irrGlFramebufferTexture2D(GL_FRAMEBUFFER, GL_STENCIL_ATTACHMENT, GL_TEXTURE_2D, 0, 0);
+							Driver->irrGlFramebufferTexture2D(GL_FRAMEBUFFER, GL_STENCIL_ATTACHMENT, textarget, 0, 0);
 
 						AssignedStencil = false;
 					}
 #endif
 					AssignedDepth = true;
 				} else {
+					// No (valid) depth/stencil texture.
 					if (AssignedDepth)
 						Driver->irrGlFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_TEXTURE_2D, 0, 0);
 

irr/src/COpenGLCoreTexture.h

@@ -45,12 +45,13 @@ class COpenGLCoreTexture : public ITexture
 
 	COpenGLCoreTexture(const io::path &name, const std::vector<IImage *> &srcImages, E_TEXTURE_TYPE type, TOpenGLDriver *driver) :
 			ITexture(name, type), Driver(driver), TextureType(GL_TEXTURE_2D),
-			TextureName(0), InternalFormat(GL_RGBA), PixelFormat(GL_RGBA), PixelType(GL_UNSIGNED_BYTE), Converter(0), LockReadOnly(false), LockImage(0), LockLayer(0),
+			TextureName(0), InternalFormat(GL_RGBA), PixelFormat(GL_RGBA), PixelType(GL_UNSIGNED_BYTE), MSAA(0), Converter(0), LockReadOnly(false), LockImage(0), LockLayer(0),
 			KeepImage(false), MipLevelStored(0), LegacyAutoGenerateMipMaps(false)
 	{
 		_IRR_DEBUG_BREAK_IF(srcImages.empty())
 
 		DriverType = Driver->getDriverType();
+		_IRR_DEBUG_BREAK_IF(Type == ETT_2D_MS); // not supported by this constructor
 		TextureType = TextureTypeIrrToGL(Type);
 		HasMipMaps = Driver->getTextureCreationFlag(ETCF_CREATE_MIP_MAPS);
 		KeepImage = Driver->getTextureCreationFlag(ETCF_ALLOW_MEMORY_COPY);
@@ -141,10 +142,10 @@ class COpenGLCoreTexture : public ITexture
 		TEST_GL_ERROR(Driver);
 	}
 
-	COpenGLCoreTexture(const io::path &name, const core::dimension2d<u32> &size, E_TEXTURE_TYPE type, ECOLOR_FORMAT format, TOpenGLDriver *driver) :
+	COpenGLCoreTexture(const io::path &name, const core::dimension2d<u32> &size, E_TEXTURE_TYPE type, ECOLOR_FORMAT format, TOpenGLDriver *driver, u8 msaa = 0) :
 			ITexture(name, type),
 			Driver(driver), TextureType(GL_TEXTURE_2D),
-			TextureName(0), InternalFormat(GL_RGBA), PixelFormat(GL_RGBA), PixelType(GL_UNSIGNED_BYTE), Converter(0), LockReadOnly(false), LockImage(0), LockLayer(0), KeepImage(false),
+			TextureName(0), InternalFormat(GL_RGBA), PixelFormat(GL_RGBA), PixelType(GL_UNSIGNED_BYTE), MSAA(msaa), Converter(0), LockReadOnly(false), LockImage(0), LockLayer(0), KeepImage(false),
 			MipLevelStored(0), LegacyAutoGenerateMipMaps(false)
 	{
 		DriverType = Driver->getDriverType();
@@ -184,23 +185,47 @@ class COpenGLCoreTexture : public ITexture
 		const COpenGLCoreTexture *prevTexture = Driver->getCacheHandler()->getTextureCache().get(0);
 		Driver->getCacheHandler()->getTextureCache().set(0, this);
 
-		GL.TexParameteri(TextureType, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
-		GL.TexParameteri(TextureType, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
-		GL.TexParameteri(TextureType, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-		GL.TexParameteri(TextureType, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+		// An INVALID_ENUM error is generated by TexParameter* if target is either
+		// TEXTURE_2D_MULTISAMPLE or TEXTURE_2D_MULTISAMPLE_ARRAY, and pname is any
+		// sampler state from table 23.18.
+		// ~ https://registry.khronos.org/OpenGL/specs/gl/glspec46.core.pdf
+		if (Type != ETT_2D_MS) {
+			GL.TexParameteri(TextureType, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+			GL.TexParameteri(TextureType, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+			GL.TexParameteri(TextureType, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+			GL.TexParameteri(TextureType, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 
 #if defined(GL_VERSION_1_2)
-		GL.TexParameteri(TextureType, GL_TEXTURE_WRAP_R, GL_CLAMP_TO_EDGE);
+			GL.TexParameteri(TextureType, GL_TEXTURE_WRAP_R, GL_CLAMP_TO_EDGE);
 #endif
 
-		StatesCache.WrapU = ETC_CLAMP_TO_EDGE;
-		StatesCache.WrapV = ETC_CLAMP_TO_EDGE;
-		StatesCache.WrapW = ETC_CLAMP_TO_EDGE;
+			StatesCache.WrapU = ETC_CLAMP_TO_EDGE;
+			StatesCache.WrapV = ETC_CLAMP_TO_EDGE;
+			StatesCache.WrapW = ETC_CLAMP_TO_EDGE;
+		}
 
 		switch (Type) {
 		case ETT_2D:
 			GL.TexImage2D(GL_TEXTURE_2D, 0, InternalFormat, Size.Width, Size.Height, 0, PixelFormat, PixelType, 0);
 			break;
+		case ETT_2D_MS: {
+			// glTexImage2DMultisample is supported by OpenGL 3.2+
+			// glTexStorage2DMultisample is supported by OpenGL 4.3+ and OpenGL ES 3.1+
+#ifdef IRR_COMPILE_GL_COMMON // legacy driver
+			constexpr bool use_gl_impl = true;
+#else
+			const bool use_gl_impl = Driver->Version.Spec != OpenGLSpec::ES;
+#endif
+			GLint max_samples = 0;
+			GL.GetIntegerv(GL_MAX_SAMPLES, &max_samples);
+			MSAA = std::min(MSAA, (u8)max_samples);
+
+			if (use_gl_impl)
+				GL.TexImage2DMultisample(GL_TEXTURE_2D_MULTISAMPLE, MSAA, InternalFormat, Size.Width, Size.Height, GL_TRUE);
+			else
+				GL.TexStorage2DMultisample(GL_TEXTURE_2D_MULTISAMPLE, MSAA, InternalFormat, Size.Width, Size.Height, GL_TRUE);
+			break;
+		}
 		case ETT_CUBEMAP:
 			GL.TexImage2D(GL_TEXTURE_CUBE_MAP_POSITIVE_X, 0, InternalFormat, Size.Width, Size.Height, 0, PixelFormat, PixelType, 0);
 			GL.TexImage2D(GL_TEXTURE_CUBE_MAP_NEGATIVE_X, 0, InternalFormat, Size.Width, Size.Height, 0, PixelFormat, PixelType, 0);
@@ -595,6 +620,8 @@ class COpenGLCoreTexture : public ITexture
 		switch (type) {
 		case ETT_2D:
 			return GL_TEXTURE_2D;
+		case ETT_2D_MS:
+			return GL_TEXTURE_2D_MULTISAMPLE;
 		case ETT_CUBEMAP:
 			return GL_TEXTURE_CUBE_MAP;
 		}
@@ -610,6 +637,7 @@ class COpenGLCoreTexture : public ITexture
 	GLint InternalFormat;
 	GLenum PixelFormat;
 	GLenum PixelType;
+	u8 MSAA;
 	void (*Converter)(const void *, s32, void *);
 
 	bool LockReadOnly;

irr/src/COpenGLDriver.cpp

@@ -651,6 +651,29 @@ IRenderTarget *COpenGLDriver::addRenderTarget()
 	return renderTarget;
 }
 
+void COpenGLDriver::blitRenderTarget(IRenderTarget *from, IRenderTarget *to)
+{
+	if (Version < 300) {
+		os::Printer::log("glBlitFramebuffer not supported by OpenGL < 3.0", ELL_ERROR);
+		return;
+	}
+
+	GLuint prev_fbo_id;
+	CacheHandler->getFBO(prev_fbo_id);
+
+	COpenGLRenderTarget *src = static_cast<COpenGLRenderTarget *>(from);
+	COpenGLRenderTarget *dst = static_cast<COpenGLRenderTarget *>(to);
+	GL.BindFramebuffer(GL.READ_FRAMEBUFFER, src->getBufferID());
+	GL.BindFramebuffer(GL.DRAW_FRAMEBUFFER, dst->getBufferID());
+	GL.BlitFramebuffer(
+			0, 0, src->getSize().Width, src->getSize().Height,
+			0, 0, dst->getSize().Width, dst->getSize().Height,
+			GL.COLOR_BUFFER_BIT | GL.DEPTH_BUFFER_BIT | GL.STENCIL_BUFFER_BIT, GL.NEAREST);
+
+	// This resets both read and draw framebuffer. Note that we bypass CacheHandler here.
+	GL.BindFramebuffer(GL.FRAMEBUFFER, prev_fbo_id);
+}
+
 // small helper function to create vertex buffer object address offsets
 static inline const GLvoid *buffer_offset(const size_t offset)
 {
@@ -2091,7 +2114,9 @@ void COpenGLDriver::setBasicRenderStates(const SMaterial &material, const SMater
 		else if (lastmaterial.AntiAliasing & EAAM_ALPHA_TO_COVERAGE)
 			glDisable(GL_SAMPLE_ALPHA_TO_COVERAGE_ARB);
 
-		if ((AntiAlias >= 2) && (material.AntiAliasing & (EAAM_SIMPLE | EAAM_QUALITY))) {
+		// Enable MSAA even if it's not enabled in the OpenGL context, we might
+		// be rendering to an FBO with multisampling.
+		if (material.AntiAliasing & (EAAM_SIMPLE | EAAM_QUALITY)) {
 			glEnable(GL_MULTISAMPLE_ARB);
 #ifdef GL_NV_multisample_filter_hint
 			if (FeatureAvailable[IRR_NV_multisample_filter_hint]) {
@@ -2694,6 +2719,12 @@ IVideoDriver *COpenGLDriver::getVideoDriver()
 
 ITexture *COpenGLDriver::addRenderTargetTexture(const core::dimension2d<u32> &size,
 		const io::path &name, const ECOLOR_FORMAT format)
+{
+	return addRenderTargetTextureMs(size, 0, name, format);
+}
+
+ITexture *COpenGLDriver::addRenderTargetTextureMs(const core::dimension2d<u32> &size, u8 msaa,
+		const io::path &name, const ECOLOR_FORMAT format)
 {
 	if (IImage::isCompressedFormat(format))
 		return 0;
@@ -2711,7 +2742,7 @@ ITexture *COpenGLDriver::addRenderTargetTexture(const core::dimension2d<u32> &si
 		destSize = destSize.getOptimalSize((size == size.getOptimalSize()), false, false);
 	}
 
-	COpenGLTexture *renderTargetTexture = new COpenGLTexture(name, destSize, ETT_2D, format, this);
+	COpenGLTexture *renderTargetTexture = new COpenGLTexture(name, destSize, msaa > 0 ? ETT_2D_MS : ETT_2D, format, this, msaa);
 	addTexture(renderTargetTexture);
 	renderTargetTexture->drop();