allow loading controlnets from passed paths

.github/workflows/python-app.yml

@@ -29,6 +29,14 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: clone diffusers
+      uses: actions/checkout@v4
+      with:
+        repository: huggingface/diffusers
+        path: diffusers
+    - name: install diffusers
+      run: |
+         cd diffusers && python3 -m pip install .
     - name: pull lfs
       run: |
         cd tests/models-sd/SDXL/tiny-sdxl && git lfs pull

examples/cim2im_flux.py

@@ -0,0 +1,26 @@
+import torch
+from multigen.prompting import Cfgen
+from multigen.sessions import GenSession
+from multigen.pipes import Cond2ImPipe, ModelType, CIm2ImPipe
+
+
+cnet_ids = ["InstantX/FLUX.1-dev-Controlnet-Canny-alpha"]
+
+# comment this line if FLUX.1-dev-Controlnet-Canny-alpha is not saved locally
+cnet_ids = ["/home/imgen/models/ControlNetFlux/FLUX.1-dev-Controlnet-Canny-alpha/"]
+ctypes = ["canny"]
+device = torch.device('cpu')
+offload_device = 0
+model_id = '/home/imgen/models/flux-1-dev'
+pipe = CIm2ImPipe(model_id, ctypes=ctypes, cnet_ids=cnet_ids, device=device, offload_device=offload_device)
+
+
+prompt = ["bioinformatics lab with flasks and exotic flowers",
+           "happy vibrant", "green colors", "artwork", "high tech"]
+
+nprompt = "jpeg artifacts, blur, distortion, watermark, extra fingers, fewer fingers, lowres, bad hands, duplicate heads, bad anatomy"
+
+pipe.setup("./_projects/biolab/00000.png", strength=1, steps=25, cscales=0.8)
+
+gs = GenSession("./_projects/biolab/controlnet-flux/", pipe, Cfgen(prompt, nprompt))
+gs.gen_sess(add_count=10)

examples/flux_inp_meta.py

@@ -0,0 +1,21 @@
+import torch
+from diffusers import FluxControlNetModel, FluxControlNetInpaintPipeline 
+from diffusers import FluxInpaintPipeline 
+from multigen.pipes import InpaintingPipe
+from diffusers.utils import load_image, make_image_grid
+
+# requires https://huggingface.co/black-forest-labs/FLUX.1-dev
+model_id = "/home/imgen/models/flux-1-dev"
+pipe = InpaintingPipe(model_id, 
+        device=torch.device('cpu', 0), offload_device=0, torch_dtype=torch.bfloat16)
+
+# load base and mask image
+init_image = load_image("cr.png")
+mask_image = load_image("select.png")
+
+generator = torch.Generator("cuda").manual_seed(92)
+prompt = "a football player holding a gun, pointing it towards viewer"
+pipe.setup(image=init_image, mask=mask_image, guidance_scale=7,
+        strength=0.9, steps=30)
+image = pipe.gen(dict(prompt=prompt, generator=generator))
+image.save('result0.png')

examples/inp2im.py

@@ -1,6 +1,6 @@
 from multigen.prompting import Cfgen
 from multigen.sessions import GenSession
-from multigen.pipes import InpaintingPipe
+from multigen.pipes import CInpaintingPipe
 
 from PIL import Image
 
@@ -12,7 +12,7 @@
           "happy vibrant", "green colors", "artwork", "high tech"]
 
 mask_image = Image.open("mask_up.png").resize((768, 768))
-pipe = InpaintingPipe(model_dir+model_id)
+pipe = CInpaintingPipe(model_dir+model_id)
 pipe.setup("./_projects/biolab/00000.png", mask_image)
 gs = GenSession("./_projects/biolab/inpaint/", pipe, Cfgen(prompt, nprompt))
 gs.gen_sess(add_count=5)

examples/sketch_inpaint.py

@@ -16,7 +16,7 @@ def main():
     scheduler = "EulerAncestralDiscreteScheduler" # gives good results
 
     pipe = MaskedIm2ImPipe(model_dir+model_id)
-    pipe.setup(original_image=img, image_painted=img_paint, strength=0.85,
+    pipe.setup(image=img, image_painted=img_paint, strength=0.85,
                scheduler=scheduler, guidance_scale=7, clip_skip=3, blur=blur)
 
     prompt = "a man wearing a mask"

examples/sketch_inpaint_xl.py

@@ -15,7 +15,7 @@ def main():
 
     pipe = MaskedIm2ImPipe(model_id, model_type=ModelType.SDXL)
     blur = 48
-    pipe.setup(original_image=img, image_painted=img_paint, strength=0.96,
+    pipe.setup(image=img, image_painted=img_paint, strength=0.96,
                scheduler=scheduler, guidance_scale=7, clip_skip=0, blur=blur, blur_compose=3, steps=50, sample_mode='random')
 
     prompt = "a man wearing a mask"