Add canny and gaussian blur filters

lagadic · May 3, 2024 · be943e8 · be943e8
1 parent 2e68182
commit be943e8
Show file tree

Hide file tree

Showing 7 changed files with 261 additions and 34 deletions.
diff --git a/modules/ar/include/visp3/ar/vpPanda3DCommonFilters.h b/modules/ar/include/visp3/ar/vpPanda3DCommonFilters.h
@@ -9,16 +9,54 @@
 
 class vpPanda3DRGBRenderer;
 
+/**
+ * @brief Class that implements an RGB to grayscale conversion.
+ *
+ */
 class VISP_EXPORT vpPanda3DLuminanceFilter : public vpPanda3DPostProcessFilter
 {
 public:
-  vpPanda3DLuminanceFilter(const std::string &name, std::shared_ptr<vpPanda3DRGBRenderer> &inputRenderer, bool isOutput);
+  vpPanda3DLuminanceFilter(const std::string &name, std::shared_ptr<vpPanda3DRGBRenderer> inputRenderer, bool isOutput);
   FrameBufferProperties getBufferProperties() const vp_override;
   void getRender(vpImage<unsigned char> &I) const;
 
 private:
   static const char *FRAGMENT_SHADER;
+};
+
+/**
+ * @brief Class that implements a gaussian filter on a grayscale image.
+ * The grayscale image should be contained in the blue channel of the image.
+ *
+ */
+class VISP_EXPORT vpPanda3DGaussianBlur : public vpPanda3DPostProcessFilter
+{
+public:
+  vpPanda3DGaussianBlur(const std::string &name, std::shared_ptr<vpPanda3DBaseRenderer> inputRenderer, bool isOutput);
+  FrameBufferProperties getBufferProperties() const vp_override;
+  void getRender(vpImage<unsigned char> &I) const;
+
+private:
+  static const char *FRAGMENT_SHADER;
+};
 
+
+class VISP_EXPORT vpPanda3DCanny : public vpPanda3DPostProcessFilter
+{
+public:
+  vpPanda3DCanny(const std::string &name, std::shared_ptr<vpPanda3DBaseRenderer> inputRenderer, bool isOutput, float edgeThreshold);
+  FrameBufferProperties getBufferProperties() const vp_override;
+  void getRender(vpImage<vpRGBf> &I) const;
+  void setEdgeThreshold(float edgeThreshold);
+
+protected:
+  void setupScene() vp_override;
+
+private:
+  static const char *FRAGMENT_SHADER;
+  float m_edgeThreshold;
 };
+
+
 #endif
 #endif
diff --git a/modules/ar/include/visp3/ar/vpPanda3DPostProcessFilter.h b/modules/ar/include/visp3/ar/vpPanda3DPostProcessFilter.h
@@ -12,7 +12,7 @@
 class VISP_EXPORT vpPanda3DPostProcessFilter : public vpPanda3DBaseRenderer
 {
 public:
-  vpPanda3DPostProcessFilter(const std::string &name, const std::shared_ptr<vpPanda3DBaseRenderer> &inputRenderer, bool isOutput, std::string fragmentShader)
+  vpPanda3DPostProcessFilter(const std::string &name, std::shared_ptr<vpPanda3DBaseRenderer> inputRenderer, bool isOutput, std::string fragmentShader)
     : vpPanda3DBaseRenderer(name), m_inputRenderer(inputRenderer), m_isOutput(isOutput), m_fragmentShader(fragmentShader)
   {
     m_renderOrder = m_inputRenderer->getRenderOrder() + 1;
@@ -22,8 +22,10 @@ class VISP_EXPORT vpPanda3DPostProcessFilter : public vpPanda3DBaseRenderer
     return false;
   }
 
+  GraphicsOutput *getMainOutputBuffer() vp_override { return m_buffer; }
+
 protected:
-  void setupScene() vp_override;
+  virtual void setupScene() vp_override;
 
   void setupCamera() vp_override;
 
@@ -36,7 +38,7 @@ class VISP_EXPORT vpPanda3DPostProcessFilter : public vpPanda3DBaseRenderer
   std::shared_ptr<vpPanda3DBaseRenderer> m_inputRenderer;
   bool m_isOutput; //! Whether this filter is an output to be used and should be copied to ram
   std::string m_fragmentShader;
-  PT(Shader) shader;
+  PT(Shader) m_shader;
   Texture *m_texture;
   GraphicsOutput *m_buffer;
 

diff --git a/modules/ar/src/panda3d-simulator/vpPanda3DCommonFilters.cpp b/modules/ar/src/panda3d-simulator/vpPanda3DCommonFilters.cpp
@@ -20,26 +20,93 @@ void main() {
 )shader";
 
 
-vpPanda3DLuminanceFilter::vpPanda3DLuminanceFilter(const std::string &name, std::shared_ptr<vpPanda3DRGBRenderer> &inputRenderer, bool isOutput)
+vpPanda3DLuminanceFilter::vpPanda3DLuminanceFilter(const std::string &name, std::shared_ptr<vpPanda3DRGBRenderer> inputRenderer, bool isOutput)
   : vpPanda3DPostProcessFilter(name, inputRenderer, isOutput, std::string(vpPanda3DLuminanceFilter::FRAGMENT_SHADER))
+{ }
+FrameBufferProperties vpPanda3DLuminanceFilter::getBufferProperties() const
 {
+  FrameBufferProperties fbp;
+  fbp.set_depth_bits(0);
+  fbp.set_rgba_bits(0, 0, 8, 0);
+  fbp.set_float_color(false);
+  return fbp;
+}
+void vpPanda3DLuminanceFilter::getRender(vpImage<unsigned char> &I) const
+{
+  if (!m_isOutput) {
+    throw vpException(vpException::fatalError, "Tried to fetch output of a postprocessing filter that was configured as an intermediate output");
+  }
+  unsigned indexMultiplier = m_texture->get_num_components(); // we ask for only 8 bits image, but we may get an rgb image
+  I.resize(m_renderParameters.getImageHeight(), m_renderParameters.getImageWidth());
+  unsigned char *data = (unsigned char *)(&(m_texture->get_ram_image().front()));
+  if (indexMultiplier != 1) {
+    for (unsigned int i = 0; i < I.getSize(); ++i) {
+      I.bitmap[i] = data[i * indexMultiplier];
+    }
+  }
+  else {
+    memcpy(I.bitmap, &data[0], I.getSize() * sizeof(unsigned char));
+  }
+}
+
+
+const char *vpPanda3DGaussianBlur::FRAGMENT_SHADER = R"shader(
+#version 330
+
+in vec2 texcoords;
+
+uniform sampler2D p3d_Texture0;
+uniform vec2 dp; // 1 divided by number of pixels
+
+const float kernel[25] = float[25](
+  2, 4, 5, 4, 2,
+  4, 9, 12, 9, 4,
+  5, 12, 15, 12, 5,
+  4, 9, 12, 9, 4,
+  2, 4, 5, 4, 2
+);
+const float normalize = 1 / 159.0;
+
+vec2 offset[25] = vec2[25](
+  vec2(-2*dp.x,-2*dp.y),  vec2(-dp.x,-2*dp.y),  vec2(0,-2*dp.y),    vec2(dp.x,-2*dp.y), vec2(2*dp.x,-2*dp.y),
+  vec2(-2*dp.x,-dp.y),    vec2(-dp.x, -dp.y),   vec2(0.0, -dp.y),   vec2(dp.x, -dp.y),  vec2(2*dp.x,-dp.y),
+  vec2(-2*dp.x,0.0),      vec2(-dp.x, 0.0),     vec2(0.0, 0.0),     vec2(dp.x, 0.0),    vec2(2*dp.x,0.0),
+  vec2(-2*dp.x, dp.y),    vec2(-dp.x, dp.y),    vec2(0.0, dp.y),    vec2(dp.x, dp.y),   vec2(2*dp.x, dp.y),
+  vec2(-2*dp.x, 2*dp.y),  vec2(-dp.x, 2*dp.y),  vec2(0.0, 2*dp.y),  vec2(dp.x, 2*dp.y), vec2(2*dp.x, 2*dp.y)
+);
 
+out vec4 p3d_FragData;
+
+void main() {
+  float v = 0.f;
+
+  for(int i = 0; i < 25; ++i) {
+    v += kernel[i] * texture(p3d_Texture0, texcoords + offset[i]).b ;
+  }
+  p3d_FragData.b = v * normalize;
 }
-FrameBufferProperties vpPanda3DLuminanceFilter::getBufferProperties() const
+)shader";
+
+vpPanda3DGaussianBlur::vpPanda3DGaussianBlur(const std::string &name, std::shared_ptr<vpPanda3DBaseRenderer> inputRenderer, bool isOutput)
+  : vpPanda3DPostProcessFilter(name, inputRenderer, isOutput, vpPanda3DGaussianBlur::FRAGMENT_SHADER)
+{ }
+
+FrameBufferProperties vpPanda3DGaussianBlur::getBufferProperties() const
 {
   FrameBufferProperties fbp;
   fbp.set_depth_bits(0);
   fbp.set_rgba_bits(0, 0, 8, 0);
   fbp.set_float_color(false);
   return fbp;
 }
-void vpPanda3DLuminanceFilter::getRender(vpImage<unsigned char> &I) const
+
+void vpPanda3DGaussianBlur::getRender(vpImage<unsigned char> &I) const
 {
   if (!m_isOutput) {
     throw vpException(vpException::fatalError, "Tried to fetch output of a postprocessing filter that was configured as an intermediate output");
   }
   unsigned indexMultiplier = m_texture->get_num_components(); // we ask for only 8 bits image, but we may get an rgb image
-  I.resize(m_texture->get_y_size(), m_texture->get_x_size());
+  I.resize(m_renderParameters.getImageHeight(), m_renderParameters.getImageWidth());
   unsigned char *data = (unsigned char *)(&(m_texture->get_ram_image().front()));
   if (indexMultiplier != 1) {
     for (unsigned int i = 0; i < I.getSize(); ++i) {
@@ -51,6 +118,104 @@ void vpPanda3DLuminanceFilter::getRender(vpImage<unsigned char> &I) const
   }
 }
 
+const char *vpPanda3DCanny::FRAGMENT_SHADER = R"shader(
+#version 330
+
+in vec2 texcoords;
+
+uniform sampler2D p3d_Texture0;
+uniform vec2 dp; // 1 divided by number of pixels
+uniform float edgeThreshold;
+
+const float kernel[9] = float[9](
+  0.0, 1.0, 0.0,
+  1.0,-4.0, 1.0,
+  0.0, 1.0, 0.0
+);
+
+const float kernel_h[9] = float[9](
+  -1.0, 0.0, 1.0,
+  -2.0, 0.0, 2.0,
+  -1.0, 0.0, 1.0
+);
+
+const float kernel_v[9] = float[9](
+  -1.0, -2.0, -1.0,
+  0.0, 0.0, 0.0,
+  1.0, 2.0, 1.0
+);
+
+vec2 offset[9] = vec2[9](
+  vec2(-dp.x, -dp.y), vec2(0.0, -dp.y), vec2(dp.x, -dp.y),
+  vec2(-dp.x, 0.0),   vec2(0.0, 0.0),   vec2(dp.x, 0.0),
+  vec2(-dp.x, dp.y),  vec2(0.0, dp.y),  vec2(dp.x, dp.y)
+);
+
+
+out vec4 p3d_FragData;
+
+void main() {
+  float sum = 0.f;
+  for(int i = 0; i < 9; ++i) {
+    float pix = texture(p3d_Texture0, texcoords + offset[i]).b;
+    sum += pix * kernel[i];
+  }
+  if(abs(sum * 255.f) > edgeThreshold) {
+    float sum_h = 0.f;
+    float sum_v = 0.f;
+    for(int i = 0; i < 9; ++i) {
+      float pix = texture(p3d_Texture0, texcoords + offset[i]).b;
+      sum_h += pix * kernel_h[i];
+      sum_v += pix * kernel_v[i];
+    }
+
+    vec2 orientationAndValid = sum_h * sum_h + sum_v *  sum_v > 0 ? vec2(atan(sum_v/sum_h), 1.f) : vec2(0.f, 0.f);
+    p3d_FragData = vec4(sum_h, sum_v, orientationAndValid.x, orientationAndValid.y);
+  } else {
+    p3d_FragData = vec4(0.f, 0.f, 0.f, 0.f);
+  }
+}
+)shader";
+
+vpPanda3DCanny::vpPanda3DCanny(const std::string &name, std::shared_ptr<vpPanda3DBaseRenderer> inputRenderer, bool isOutput, float edgeThreshold)
+  : vpPanda3DPostProcessFilter(name, inputRenderer, isOutput, vpPanda3DCanny::FRAGMENT_SHADER), m_edgeThreshold(edgeThreshold)
+{ }
+
+void vpPanda3DCanny::setupScene()
+{
+  vpPanda3DPostProcessFilter::setupScene();
+  m_renderRoot.set_shader_input("edgeThreshold", LVector2f(m_edgeThreshold));
+}
 
+void vpPanda3DCanny::setEdgeThreshold(float edgeThreshold)
+{
+  m_edgeThreshold = edgeThreshold;
+  m_renderRoot.set_shader_input("edgeThreshold", LVector2f(m_edgeThreshold));
+}
+
+
+FrameBufferProperties vpPanda3DCanny::getBufferProperties() const
+{
+  FrameBufferProperties fbp;
+  fbp.set_depth_bits(0);
+  fbp.set_rgba_bits(32, 32, 32, 32);
+  fbp.set_float_color(true);
+  return fbp;
+}
+
+void vpPanda3DCanny::getRender(vpImage<vpRGBf> &I) const
+{
+  if (!m_isOutput) {
+    throw vpException(vpException::fatalError, "Tried to fetch output of a postprocessing filter that was configured as an intermediate output");
+  }
+  unsigned indexMultiplier = m_texture->get_num_components(); // we ask for only 8 bits image, but we may get an rgb image
+  I.resize(m_renderParameters.getImageHeight(), m_renderParameters.getImageWidth());
+  float *data = (float *)(&(m_texture->get_ram_image().front()));
+  for (unsigned int i = 0; i < I.getSize(); ++i) {
+    I.bitmap[i].B = (data[i * 4]);
+    I.bitmap[i].G = (data[i * 4 + 1]);
+    I.bitmap[i].R = (data[i * 4 + 2]);
+  }
+}
 
 #endif
diff --git a/modules/ar/src/panda3d-simulator/vpPanda3DPostProcessFilter.cpp b/modules/ar/src/panda3d-simulator/vpPanda3DPostProcessFilter.cpp
@@ -30,10 +30,11 @@ void vpPanda3DPostProcessFilter::setupScene()
     throw vpException(vpException::fatalError,
     "Cannot add a postprocess filter to a renderer that does not define getMainOutputBuffer()");
   }
-  shader = Shader::make(Shader::ShaderLanguage::SL_GLSL,
+  m_shader = Shader::make(Shader::ShaderLanguage::SL_GLSL,
                         FILTER_VERTEX_SHADER,
                         m_fragmentShader);
-  m_renderRoot.set_shader(shader);
+  m_renderRoot.set_shader(m_shader);
+  m_renderRoot.set_shader_input("dp", LVector2f(1.0 / buffer->get_texture()->get_x_size(), 1.0 / buffer->get_texture()->get_y_size()));
   std::cout << m_fragmentShader << std::endl;
   m_renderRoot.set_texture(buffer->get_texture());
   m_renderRoot.set_attrib(LightRampAttrib::make_identity());
@@ -77,7 +78,6 @@ void vpPanda3DPostProcessFilter::setupRenderTarget()
   m_buffers.push_back(m_buffer);
   m_buffer->set_inverted(gsg->get_copy_texture_inverted());
   m_texture = new Texture();
-  //fbp.setup_color_texture(m_texture);
   m_buffer->add_render_texture(m_texture, m_isOutput ? GraphicsOutput::RenderTextureMode::RTM_copy_ram : GraphicsOutput::RenderTextureMode::RTM_copy_texture);
   m_buffer->set_clear_color(LColor(0.f));
   m_buffer->set_clear_color_active(true);
@@ -96,7 +96,10 @@ void vpPanda3DPostProcessFilter::setRenderParameters(const vpPanda3DRenderParame
   bool resize = previousH != params.getImageHeight() || previousW != params.getImageWidth();
 
   m_renderParameters = params;
-
+  if (m_window != nullptr) {
+    GraphicsOutput *buffer = m_inputRenderer->getMainOutputBuffer();
+    m_renderRoot.set_shader_input("dp", LVector2f(1.0 / buffer->get_texture()->get_x_size(), 1.0 / buffer->get_texture()->get_y_size()));
+  }
   if (resize) {
     for (GraphicsOutput *buffer: m_buffers) {
       //buffer->get_type().is_derived_from()

diff --git a/modules/ar/src/panda3d-simulator/vpPanda3DRGBRenderer.cpp b/modules/ar/src/panda3d-simulator/vpPanda3DRGBRenderer.cpp
@@ -85,7 +85,6 @@ const char *vpPanda3DRGBRenderer::COOK_TORRANCE_FRAG = R"shader(
 // Version 330, specified when generating shader
 #define M_PI 3.1415926535897932384626433832795
 
-
 in vec3 oNormal;
 in vec4 viewVertex;
 in vec3 F0;
@@ -138,8 +137,6 @@ float G(float hn, float nv, float nl, float vh)
   return min(1.0, min((2.f * hn * nv) / vh, (2.f * hn * nl) / vh));
 }
 
-
-
 vec3 F(vec3 F0, float vh)
 {
   return F0 + (vec3(1.f, 1.f, 1.f) - F0) * pow(1.f - vh, 5);
@@ -151,14 +148,16 @@ void main()
   vec3 n = normalize(oNormal); // normalized normal vector
   vec3 v = normalize(-viewVertex.xyz); // normalized view vector
   float nv = max(0.f, dot(n, v));
-  float roughness2 = pow(p3d_Material.roughness, 2);
+  float roughness2 = clamp(pow(p3d_Material.roughness, 2), 0.01, 0.99);
 
   #ifdef HAS_TEXTURE
     vec4 baseColor = texture(p3d_Texture0, texcoords);
+    vec4 ambientColor = baseColor;
   #else
+    vec4 ambientColor = p3d_Material.ambient;
     vec4 baseColor = p3d_Material.baseColor;
   #endif
-  //p3d_FragData = vec4(0.0, 0.0, 0.0, 1.f);
+
   p3d_FragData = p3d_LightModel.ambient * baseColor;
 
 
@@ -176,9 +175,8 @@ void main()
     float attenuation = 1.f / (aFac[0] + aFac[1] * lightDist + aFac[2] * lightDist * lightDist);
 
     vec3 FV = F(F0, vh);
-    vec3 kd = (1.f - p3d_Material.metallic) * (1.f - FV) * (1.f / M_PI);
+    vec3 kd = (1.f - p3d_Material.metallic) *  (1.f - FV) * (1.f / M_PI);
 
-    vec4 diffuseColor = baseColor;
 
     #ifdef SPECULAR
       vec3 specularColor = vec3(0.f, 0.f, 0.f);
@@ -192,7 +190,7 @@ void main()
       vec3 specularColor = vec3(0.0, 0.0, 0.0);
     #endif
 
-    p3d_FragData += (p3d_LightSource[i].color * attenuation) * nl * (diffuseColor * vec4(kd, 1.f) + vec4(specularColor, 1.f));
+    p3d_FragData += (p3d_LightSource[i].color * attenuation) * nl * (baseColor * vec4(kd, 1.f) + vec4(specularColor, 1.f));
   }
 }
 )shader";
@@ -219,10 +217,12 @@ void vpPanda3DRGBRenderer::addNodeToScene(const NodePath &object)
 {
   NodePath objectInScene = object.copy_to(m_renderRoot);
   objectInScene.set_name(object.get_name());
+  const RenderAttrib *textureAttrib = objectInScene.get_attrib(TextureAttrib::get_class_type());
+  bool hasTexture = textureAttrib != nullptr;
   std::cout << "SHOW SPECULARS = " << m_showSpeculars << std::endl;
   PT(Shader) shader = Shader::make(Shader::ShaderLanguage::SL_GLSL,
                                     COOK_TORRANCE_VERT,
-                                    makeFragmentShader(true, m_showSpeculars));
+                                    makeFragmentShader(hasTexture, m_showSpeculars));
 
   objectInScene.set_shader(shader);