Michael/ir 1983 registry - Post Processing and Highlight (#10313)

* added color average effect

* added the outline effect to the post processing

* added smaa effect for post processing

* added ChromaticAberrationEffect to post processing

* added DotScreenEffect post processing effect

* added TiltShiftEffect post processing effect

* Added motion blur post processing effect

* added Glitch Effect to post processing

* added lineWidth GodRaysEffect and GridEffect to post processing

* added LUT1DEffect to post processing

* added LUT3DEffect post processing effect

* added LUT1DEffect post processing effect

* added NoiseEffect to post processing

* added the PixelationEffect to post processing

* added ScanlineEffect to post processing effects

* added FXAAEffect and ShockWaveEffect to post processing

* added TextureEffect to post processing effects

* added LensDistortionEffect to post processing

* corrected error with duplicate case option

* moved the effects composer code into the post processing component so we can use texture loading hooks

* committing latest progress for the post processing effects

* updated the post processing editor to use the lut path

* example

* moved the effect logic into individual use effects

* added effect options conditional check

* removed the associative array map to the effects class since it is not being used in a for loop any longer

* connected the post processing effects up to the individual useEffect

* removed the effectMap refs that are no longer needed

* updated default values for post processing effects
commented out SSGIEffect since it required the composure before it is available

* created child reactor to wait for the useScene to return a render Entity

* moving composure declaration so it can be used in an effect constructor for SSREffect

* alphabetized the effects in the editor

* added missing SSGIEffect ref

* fix render pass

* added highlights to editor items that are selected

* Added highlight system to add highlights to selected objects

* implemented the outline effect for the highlight selection
implemented the smaa default effect
removed the outline and smaa effects from the post processing editor options since they are always on

* updated the outline effect ref

* put the highlight query map ref back to correct typescript issue

* initial attempt at unity test

* updated the unity test to loop through the effects of the post processing and set its various properties using a master data obj

* Move postprocessing

* added unit test for the effect composer

* Updated the composure ref to use a factory function for the initial state of this reactor, so it stops creating a new composer and render pass every time the renderer reactor re-runs

* removed the query in the body of a reactor
removal of the highlight component, is now down by returning a cleanup function in the use effect

* updated highlight state and play mode connection to rely on the data of the selection state

* removed obsolete commented out code

* updated the post processesing component to not have a use effect for each effect

* corrected effect schema issue

* removed dev logs

* updated the post processing to avoid adding extra passes

* removed dev logs

* removed unwanted entry from the tsconfig

* corrected lint issues

* removed configure effect composure since it is no longer used

* updated to fix lint errors

* corrected the lint error

* Add SMAAEffect to default

* I fixed badly

* Implement registry method from @josh

* separated effect reactor to be it's own file
created file to populate effects registry
Added bloom effect

* added Brightness Contrast Effect

* added color average effect

* added color depth effect

* added depth of field effect
added dot screen effect
added fxaa effect
added Glitch effect

* added grid effect
added hue saturation effect

* added LUT1D Effect
added LUT3D Effect
added Lens Distortion Effect
added Linear tos RGB Effect
added Motion Blur Effect
added Noise Effect
added Outline Effect
added Pixelation Effect
added Scan line Effect

* removed effects that are completed from list

* added smaa, ssao, ssr, and shockwave effects

* added texture effect
restructured the initialization of the effects population

* added effects: TRAA, Tilt Shift, Tone mapping, Vignette

* corrected the LUT1d and LUT 3d effects

* passed down the scene and composer refs for the effects that needed them

* re added the highlight /outline effect on selected items

* updated the unit test for post processing

* removed premature return so the highlight effect will work

* add postprocessing to spatial deps

* corrected lint errors

* corrected error caught in npm check errors

* addressed PR change requests

* corrected missing ref type error

* corrected type issue

---------

Co-authored-by: Josh Field <[email protected]>
Co-authored-by: Michael Estes <[email protected]>
Co-authored-by: Daniel Belmes <[email protected]>
Co-authored-by: Daniel Belmes <[email protected]>

packages/ecs/src/ComponentFunctions.ts

@@ -132,7 +132,7 @@ export interface ComponentPartial<
    * `@todo` Explain what reactive is in this context
    * `@todo` Explain this function
    */
-  reactor?: React.FC
+  reactor?: any // previously <React.FC> breaks types
   /**
    * @todo Explain ComponentPartial.errors[]
    */

packages/editor/package.json

@@ -43,6 +43,7 @@
     "i18next": "21.6.16",
     "lodash": "4.17.21",
     "potpack": "^2.0.0",
+    "postprocessing": "6.34.1",
     "rc-dock": "3.2.18",
     "rc-slider": "10.5.0",
     "react": "18.2.0",

packages/editor/src/EditorModule.ts

@@ -28,6 +28,7 @@ import { RenderInfoSystem } from '@etherealengine/spatial/src/renderer/RenderInf
 import { EditorInstanceNetworkingSystem } from './components/realtime/EditorInstanceNetworkingSystem'
 import { EditorControlSystem } from './systems/EditorControlSystem'
 import { GizmoSystem } from './systems/GizmoSystem'
+import { HighlightSystem } from './systems/HighlightSystem'
 import { ModelHandlingSystem } from './systems/ModelHandlingSystem'
 import { ObjectGridSnapSystem } from './systems/ObjectGridSnapSystem'
 import { UploadRequestSystem } from './systems/UploadRequestSystem'
@@ -36,6 +37,7 @@ export {
   EditorInstanceNetworkingSystem,
   EditorControlSystem,
   GizmoSystem,
+  HighlightSystem,
   ModelHandlingSystem,
   ObjectGridSnapSystem,
   UploadRequestSystem,

packages/editor/src/components/toolbar/tools/PlayModeTool.tsx

@@ -47,6 +47,7 @@ import { ComputedTransformComponent } from '@etherealengine/spatial/src/transfor
 
 import { TransformGizmoControlledComponent } from '../../../classes/TransformGizmoControlledComponent'
 import { EditorState } from '../../../services/EditorServices'
+import { SelectionState } from '../../../services/SelectionServices'
 import { transformGizmoControlledQuery } from '../../../systems/GizmoSystem'
 import { InfoTooltip } from '../../layout/Tooltip'
 import * as styles from '../styles.module.scss'
@@ -91,6 +92,9 @@ const PlayModeTool = () => {
       visualScriptQuery().forEach((entity) => dispatchAction(VisualScriptActions.execute({ entity })))
       transformGizmoControlledQuery().forEach((entity) => removeComponent(entity, TransformGizmoControlledComponent))
       //just remove all gizmo in the scene
+
+      //just clear selection to remove all higlights in the scene
+      SelectionState.updateSelection([])
     }
   }
 

packages/editor/src/functions/EditorControlFunctions.ts

@@ -49,7 +49,6 @@ import { ComponentJsonType } from '@etherealengine/engine/src/scene/types/SceneT
 import { dispatchAction, getMutableState, getState } from '@etherealengine/hyperflux'
 import { DirectionalLightComponent, HemisphereLightComponent } from '@etherealengine/spatial'
 import { MAT4_IDENTITY } from '@etherealengine/spatial/src/common/constants/MathConstants'
-import { PostProcessingComponent } from '@etherealengine/spatial/src/renderer/components/PostProcessingComponent'
 import { VisibleComponent } from '@etherealengine/spatial/src/renderer/components/VisibleComponent'
 import { getMaterial } from '@etherealengine/spatial/src/renderer/materials/materialFunctions'
 import {
@@ -61,6 +60,7 @@ import {
 import { TransformComponent } from '@etherealengine/spatial/src/transform/components/TransformComponent'
 import { computeTransformMatrix } from '@etherealengine/spatial/src/transform/systems/TransformSystem'
 
+import { PostProcessingComponent } from '@etherealengine/spatial/src/renderer/components/PostProcessingComponent'
 import { EditorHelperState } from '../services/EditorHelperState'
 import { EditorState } from '../services/EditorServices'
 import { SelectionState } from '../services/SelectionServices'

packages/editor/src/systems/HighlightSystem.ts

@@ -0,0 +1,53 @@
+/*
+CPAL-1.0 License
+
+The contents of this file are subject to the Common Public Attribution License
+Version 1.0. (the "License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE.
+The License is based on the Mozilla Public License Version 1.1, but Sections 14
+and 15 have been added to cover use of software over a computer network and 
+provide for limited attribution for the Original Developer. In addition, 
+Exhibit A has been modified to be consistent with Exhibit B.
+
+Software distributed under the License is distributed on an "AS IS" basis,
+WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the
+specific language governing rights and limitations under the License.
+
+The Original Code is Ethereal Engine.
+
+The Original Developer is the Initial Developer. The Initial Developer of the
+Original Code is the Ethereal Engine team.
+
+All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 
+Ethereal Engine. All Rights Reserved.
+*/
+
+import { removeComponent, setComponent } from '@etherealengine/ecs'
+import { defineSystem } from '@etherealengine/ecs/src/SystemFunctions'
+import { AnimationSystemGroup } from '@etherealengine/ecs/src/SystemGroups'
+import { HighlightComponent } from '@etherealengine/spatial/src/renderer/components/HighlightComponent'
+import { useEffect } from 'react'
+import { SelectionState } from '../services/SelectionServices'
+
+const reactor = () => {
+  const selectedEntities = SelectionState.useSelectedEntities()
+
+  useEffect(() => {
+    if (!selectedEntities) return
+    const lastSelection = selectedEntities[selectedEntities.length - 1]
+    if (!lastSelection) return
+    setComponent(lastSelection, HighlightComponent)
+    return () => {
+      removeComponent(lastSelection, HighlightComponent)
+    }
+  }, [selectedEntities])
+
+  return null
+}
+
+export const HighlightSystem = defineSystem({
+  uuid: 'ee.editor.HighlightSystem',
+  insert: { with: AnimationSystemGroup },
+  reactor
+})

packages/engine/src/EngineModule.ts

@@ -31,6 +31,7 @@ export * from './avatar/AvatarModule'
 export * from './interaction/InteractionModule'
 export * from './interaction/MediaModule'
 export * from './mocap/MocapModule'
+export * from './postprocessing/PopulateEffectRegistry'
 export * from './recording/RecordingModule'
 export * from './scene/SceneModule'
 export * from './visualscript/VisualScriptModule'

packages/engine/src/postprocessing/BloomEffect.tsx

@@ -0,0 +1,114 @@
+/*
+CPAL-1.0 License
+
+The contents of this file are subject to the Common Public Attribution License
+Version 1.0. (the "License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE.
+The License is based on the Mozilla Public License Version 1.1, but Sections 14
+and 15 have been added to cover use of software over a computer network and 
+provide for limited attribution for the Original Developer. In addition, 
+Exhibit A has been modified to be consistent with Exhibit B.
+
+Software distributed under the License is distributed on an "AS IS" basis,
+WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the
+specific language governing rights and limitations under the License.
+
+The Original Code is Ethereal Engine.
+
+The Original Developer is the Initial Developer. The Initial Developer of the
+Original Code is the Ethereal Engine team.
+
+All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 
+Ethereal Engine. All Rights Reserved.
+*/
+
+import { Entity } from '@etherealengine/ecs'
+import { getMutableState, getState, none } from '@etherealengine/hyperflux'
+import {
+  EffectReactorProps,
+  PostProcessingEffectState
+} from '@etherealengine/spatial/src/renderer/effects/EffectRegistry'
+import { BlendFunction, BloomEffect, KernelSize } from 'postprocessing'
+import React, { useEffect } from 'react'
+import { PropertyTypes } from './PostProcessingRegister'
+
+declare module 'postprocessing' {
+  interface EffectComposer {
+    BloomEffect: BloomEffect
+  }
+}
+
+const effectKey = 'BloomEffect'
+
+export const BloomEffectProcessReactor: React.FC<EffectReactorProps> = (props: {
+  isActive
+  rendererEntity: Entity
+  effectData
+  effects
+}) => {
+  const { isActive, rendererEntity, effectData, effects } = props
+  const effectState = getState(PostProcessingEffectState)
+
+  useEffect(() => {
+    if (effectData[effectKey].value) return
+    effectData[effectKey].set(effectState[effectKey].defaultValues)
+  }, [])
+
+  useEffect(() => {
+    if (!isActive?.value) {
+      if (effects[effectKey].value) effects[effectKey].set(none)
+      return
+    }
+    const eff = new BloomEffect(effectData[effectKey].value)
+    effects[effectKey].set(eff)
+    return () => {
+      effects[effectKey].set(none)
+    }
+  }, [isActive])
+
+  return null
+}
+
+export const bloomAddToEffectRegistry = () => {
+  // registers the effect
+
+  getMutableState(PostProcessingEffectState).merge({
+    [effectKey]: {
+      reactor: BloomEffectProcessReactor,
+      defaultValues: {
+        isActive: true,
+        blendFunction: BlendFunction.SCREEN,
+        kernelSize: KernelSize.LARGE,
+        luminanceThreshold: 0.9,
+        luminanceSmoothing: 0.025,
+        mipmapBlur: false,
+        intensity: 1.0,
+        radius: 0.85,
+        levels: 8
+      },
+      schema: {
+        blendFunction: { propertyType: PropertyTypes.BlendFunction, name: 'Blend Function' },
+        kernelSize: { propertyType: PropertyTypes.KernelSize, name: 'Kernel Size' },
+        intensity: { propertyType: PropertyTypes.Number, name: 'Intensity', min: 0, max: 10, step: 0.01 },
+        luminanceSmoothing: {
+          propertyType: PropertyTypes.Number,
+          name: 'Luminance Smoothing',
+          min: 0,
+          max: 1,
+          step: 0.01
+        },
+        luminanceThreshold: {
+          propertyType: PropertyTypes.Number,
+          name: 'Luminance Threshold',
+          min: 0,
+          max: 1,
+          step: 0.01
+        },
+        mipmapBlur: { propertyType: PropertyTypes.Boolean, name: 'Mipmap Blur' },
+        radius: { propertyType: PropertyTypes.Number, name: 'Resolution Scale', min: 0, max: 10, step: 0.01 },
+        levels: { propertyType: PropertyTypes.Number, name: 'Resolution Scale', min: 1, max: 10, step: 1 }
+      }
+    }
+  })
+}

packages/engine/src/postprocessing/BrightnessContrastEffect.tsx

@@ -0,0 +1,91 @@
+/*
+CPAL-1.0 License
+
+The contents of this file are subject to the Common Public Attribution License
+Version 1.0. (the "License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE.
+The License is based on the Mozilla Public License Version 1.1, but Sections 14
+and 15 have been added to cover use of software over a computer network and 
+provide for limited attribution for the Original Developer. In addition, 
+Exhibit A has been modified to be consistent with Exhibit B.
+
+Software distributed under the License is distributed on an "AS IS" basis,
+WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the
+specific language governing rights and limitations under the License.
+
+The Original Code is Ethereal Engine.
+
+The Original Developer is the Initial Developer. The Initial Developer of the
+Original Code is the Ethereal Engine team.
+
+All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 
+Ethereal Engine. All Rights Reserved.
+*/
+
+import { Entity } from '@etherealengine/ecs'
+import { getMutableState, getState, none } from '@etherealengine/hyperflux'
+import {
+  EffectReactorProps,
+  PostProcessingEffectState
+} from '@etherealengine/spatial/src/renderer/effects/EffectRegistry'
+import { BlendFunction, BrightnessContrastEffect } from 'postprocessing'
+import React, { useEffect } from 'react'
+import { PropertyTypes } from './PostProcessingRegister'
+
+declare module 'postprocessing' {
+  interface EffectComposer {
+    BrightnessContrastEffect: BrightnessContrastEffect
+  }
+}
+
+const effectKey = 'BrightnessContrastEffect'
+
+export const BrightnessContrastEffectProcessReactor: React.FC<EffectReactorProps> = (props: {
+  isActive
+  rendererEntity: Entity
+  effectData
+  effects
+}) => {
+  const { isActive, rendererEntity, effectData, effects } = props
+  const effectState = getState(PostProcessingEffectState)
+
+  useEffect(() => {
+    if (effectData[effectKey].value) return
+    effectData[effectKey].set(effectState[effectKey].defaultValues)
+  }, [])
+
+  useEffect(() => {
+    if (!isActive?.value) {
+      if (effects[effectKey].value) effects[effectKey].set(none)
+      return
+    }
+    const eff = new BrightnessContrastEffect(effectData[effectKey].value)
+    effects[effectKey].set(eff)
+    return () => {
+      effects[effectKey].set(none)
+    }
+  }, [isActive])
+
+  return null
+}
+
+export const brightnessContrastAddToEffectRegistry = () => {
+  // registers the effect
+
+  getMutableState(PostProcessingEffectState).merge({
+    [effectKey]: {
+      reactor: BrightnessContrastEffectProcessReactor,
+      defaultValues: {
+        isActive: false,
+        blendFunction: BlendFunction.SRC,
+        brightness: 0.0,
+        contrast: 0.0
+      },
+      schema: {
+        brightness: { propertyType: PropertyTypes.Number, name: 'Brightness', min: -1, max: 1, step: 0.01 },
+        contrast: { propertyType: PropertyTypes.Number, name: 'Contrast', min: -1, max: 1, step: 0.01 }
+      }
+    }
+  })
+}