pass previous frame into ControlNet for video and fix ui-config

light-and-ray · light-and-ray · commit a9b57455a831 · 2024-03-28T00:46:33.000+04:00
diff --git a/README.md b/README.md
@@ -80,7 +80,7 @@ When you press the "Apply hires fix" button, the extension regenerates the image
 - ☑️ progress bar + interrupt
 - ☑️ option for pass into hires fix automatically
 - ☑️ control net
-- pass previous frame into ControlNet for video
+- ☑️ pass previous frame into ControlNet for video
 - tiled vae
 - ☑️ "hide segment anything extention" option
 - ☑️ txt2img script
diff --git a/docs/usage.md b/docs/usage.md
@@ -96,6 +96,8 @@ To suppress flickering you can generate in little fps (e.g. 10), then interpolat
 
 You can also use [sd-webui-controlnet](https://github.com/Mikubill/sd-webui-controlnet) or [lama-cleaner](https://github.com/light-and-ray/sd-webui-lama-cleaner-masked-content) with (low denosing) extensions to increase consistency, if it fits to your scenario
 
+Also a good can be to use `Pass previous frame into ControlNet` with _IP-Adapter_, _Reference_, _Revision_, _T2IA Color_, _T2IA Style_
+
 
 ## Dedicated page
 Dedicated page (replacer tab only) is available on url `/replacer-dedicated`
diff --git a/replacer/generation_args.py b/replacer/generation_args.py
@@ -80,8 +80,9 @@ class GenerationArgs:
 
     pass_into_hires_fix_automatically: bool
     save_before_hires_fix: bool
-    hires_fix_args: HiresFixArgs
+    previous_frame_into_controlnet: list
 
+    hires_fix_args: HiresFixArgs
     cn_args: list
     soft_inpaint_args: list
 
diff --git a/replacer/inpaint.py b/replacer/inpaint.py
@@ -69,7 +69,10 @@ def inpaint(
     p.do_not_save_grid = True
     try:
         if replacer_scripts.script_controlnet and gArgs.cn_args is not None and len(gArgs.cn_args) != 0:
-            replacer_scripts.enableInpaintModeForCN(gArgs.cn_args, p)
+            previousFrame = None
+            if batch_processed:
+                previousFrame = batch_processed.images[-1]
+            replacer_scripts.enableInpaintModeForCN(gArgs, p, previousFrame)
     except Exception as e:
         errors.report(f"Error {e}", exc_info=True)
 
diff --git a/replacer/replacer_scripts.py b/replacer/replacer_scripts.py
@@ -101,18 +101,32 @@ def restoreAfterCN(origImage, gArgs: GenerationArgs, processed):
         processed.images[i] = imageOrg
 
 
-def enableInpaintModeForCN(cn_args, p):
+def enableInpaintModeForCN(gArgs, p, previousFrame):
     if IS_SD_WEBUI_FORGE: return
     global external_code
+    gArgs.cn_args = list(gArgs.cn_args)
     mask = None
 
-    for controlNetUnit in cn_args:
-        controlNetUnit = external_code.to_processing_unit(controlNetUnit)
-
-        if not controlNetUnit.enabled:
+    for i in range(len(gArgs.cn_args)):
+        gArgs.cn_args[i] = external_code.to_processing_unit(gArgs.cn_args[i])
+
+        if f"Unit {i}" in gArgs.previous_frame_into_controlnet:
+            if previousFrame:
+                print(f'Passing the previous frame CN unit {i}')
+                gArgs.cn_args[i].image = {
+                    "image": convertIntoCNImageFromat(previousFrame),
+                    # "mask": convertIntoCNImageFromat(p.image_mask),
+                }
+                gArgs.cn_args[i].enabled = True
+            else:
+                print(f'Disabling CN unit {i} for the first frame')
+                gArgs.cn_args[i].enabled = False
+                continue
+
+        if not gArgs.cn_args[i].enabled:
             continue
 
-        if not IS_SD_WEBUI_FORGE and controlNetUnit.module == 'inpaint_only':
+        if not IS_SD_WEBUI_FORGE and gArgs.cn_args[i].module == 'inpaint_only':
             if p.image_mask is not None:
                 mask = p.image_mask
                 if p.inpainting_mask_invert:
@@ -121,15 +135,15 @@ def enableInpaintModeForCN(cn_args, p):
 
             print('Use cn inpaint instead of sd inpaint')
             image = limitSizeByOneDemention(p.init_images[0], max(p.width, p.height))
-            controlNetUnit.image = {
+            gArgs.cn_args[i].image = {
                 "image": convertIntoCNImageFromat(image),
                 "mask": convertIntoCNImageFromat(mask.resize(image.size)),
             }
             p.image_mask = None
             p.inpaint_full_res = False
             p.width, p.height = image.size
-            controlNetUnit.inpaint_crop_input_image = False
-            controlNetUnit.resize_mode = external_code.ResizeMode.RESIZE
+            gArgs.cn_args[i].inpaint_crop_input_image = False
+            gArgs.cn_args[i].resize_mode = external_code.ResizeMode.RESIZE
             p.needRestoreAfterCN = True
 
 
diff --git a/replacer/ui/generate_ui.py b/replacer/ui/generate_ui.py
@@ -74,6 +74,7 @@ def generate_ui(
     clip_skip,
     pass_into_hires_fix_automatically,
     save_before_hires_fix,
+    previous_frame_into_controlnet,
 
     hf_upscaler,
     hf_steps,
@@ -225,8 +226,9 @@ def readImages(input_dir):
         clip_skip=clip_skip,
         pass_into_hires_fix_automatically=pass_into_hires_fix_automatically,
         save_before_hires_fix=save_before_hires_fix,
-        hires_fix_args=hires_fix_args,
+        previous_frame_into_controlnet=previous_frame_into_controlnet if tab_index == 3 else [],
 
+        hires_fix_args=hires_fix_args,
         cn_args=cn_args,
         soft_inpaint_args=soft_inpaint_args,
         )
diff --git a/replacer/ui/replacer_tab_ui.py b/replacer/ui/replacer_tab_ui.py
@@ -160,17 +160,30 @@ def init_tab(self, isDedicatedPage: bool):
                                 placeholder="Leave blank to save images to the default path.",
                                 info='(default is the same directory with input video. Rusult is in "output_seed" subdirectory)',
                                 elem_id="replacer_video_output_dir")
-                            gr.Markdown("To increase consistency it's better to inpaint clear "\
-                                "objects on video with good quality and enough context. "\
-                                "Your prompts need to produce consistent results\n\n"\
-                                "To suppress flickering you can generate in little fps (e.g. 10), "\
-                                "then interpolate (x2) it with ai interpolation algorithm "\
-                                "(e.g [RIFE](https://github.com/megvii-research/ECCV2022-RIFE) or "\
-                                "[frame interpolation in deforum sd-webui extension]("\
-                                "https://github.com/deforum-art/sd-webui-deforum/wiki/Upscaling-and-Frame-Interpolation))\n\n"\
-                                "You can also use [sd-webui-controlnet](https://github.com/Mikubill/sd-webui-controlnet) or "\
-                                "[lama-cleaner](https://github.com/light-and-ray/sd-webui-lama-cleaner-masked-content) with (low denosing) "\
-                                "extensions to increase consistency, if it fits to your scenario")
+                            with gr.Accordion("Help", open=False):
+                                gr.Markdown(
+                                    "To increase consistency it's better to inpaint clear "\
+                                    "objects on video with good quality and enough context. "\
+                                    "Your prompts need to produce consistent results\n\n"\
+                                    \
+                                    "To suppress flickering you can generate in little fps (e.g. 10), "\
+                                    "then interpolate (x2) it with ai interpolation algorithm "\
+                                    "(e.g [RIFE](https://github.com/megvii-research/ECCV2022-RIFE) or "\
+                                    "[frame interpolation in deforum sd-webui extension]("\
+                                    "https://github.com/deforum-art/sd-webui-deforum/wiki/Upscaling-and-Frame-Interpolation))\n\n"\
+                                    \
+                                    "You can also use [sd-webui-controlnet](https://github.com/Mikubill/sd-webui-controlnet) or "\
+                                    "[lama-cleaner](https://github.com/light-and-ray/sd-webui-lama-cleaner-masked-content) with (low denosing) "\
+                                    "extensions to increase consistency, if it fits to your scenario\n\n"\
+                                    \
+                                    "Also a good can be to use `Pass previous frame into ControlNet` "\
+                                    "with _IP-Adapter_, _Reference_, _Revision_, _T2IA Color_, _T2IA Style_"
+                                    )
+                            if replacer_scripts.script_controlnet:
+                                comp.previous_frame_into_controlnet = gr.CheckboxGroup(value=[], label='Pass previous frame into ControlNet',
+                                    choices=[f"Unit {x}" for x in range(shared.opts.control_net_unit_count)], elem_id='replacer_previous_frame_into_controlnet')
+                            else:
+                                comp.previous_frame_into_controlnet = gr.CheckboxGroup(value=[], visible=False)
                     
                     comp.cn_inputs = []
                     setCustomScriptSourceForComponents("controlnet")
@@ -337,6 +350,7 @@ def init_tab(self, isDedicatedPage: bool):
                     comp.clip_skip,
                     comp.pass_into_hires_fix_automatically,
                     comp.save_before_hires_fix,
+                    comp.previous_frame_into_controlnet,
 
                     comp.hf_upscaler,
                     comp.hf_steps,
diff --git a/scripts/replacer_api.py b/scripts/replacer_api.py
@@ -146,8 +146,9 @@ async def api_replacer_replace(data: ReplaceRequest = Body(...)) -> Any:
             clip_skip=data.clip_skip,
             pass_into_hires_fix_automatically=data.use_hires_fix,
             save_before_hires_fix=False,
-            hires_fix_args=hires_fix_args,
+            previous_frame_into_controlnet=[],
 
+            hires_fix_args=hires_fix_args,
             cn_args=cn_args,
             soft_inpaint_args=soft_inpaint_args,
             )
diff --git a/scripts/replacer_script.py b/scripts/replacer_script.py
@@ -1,3 +1,4 @@
+import copy
 import gradio as gr
 from modules import scripts, scripts_postprocessing, errors, ui_settings
 from modules.processing import Processed, StableDiffusionProcessingTxt2Img
@@ -123,6 +124,10 @@ def ui(self, is_img2img):
         ] + comp.cn_inputs \
           + comp.soft_inpaint_inputs
 
+        for i in range(len(inputs)):
+            inputs[i] = copy.copy(inputs[i])
+            inputs[i].do_not_save_to_config = True
+
         return inputs
 
     def before_process(self, p: StableDiffusionProcessingTxt2Img,
@@ -273,8 +278,9 @@ def before_process(self, p: StableDiffusionProcessingTxt2Img,
             clip_skip=clip_skip,
             pass_into_hires_fix_automatically=pass_into_hires_fix_automatically,
             save_before_hires_fix=save_before_hires_fix,
-            hires_fix_args=hires_fix_args,
+            previous_frame_into_controlnet=[],
 
+            hires_fix_args=hires_fix_args,
             cn_args=cn_args,
             soft_inpaint_args=soft_inpaint_args,
             )
