Merge remote-tracking branch 'upstream/main'

# Conflicts:
#	modules/async_worker.py
#	requirements_versions.txt
#	webui.py

modules/async_worker.py

@@ -28,19 +28,19 @@ def worker():
     import modules.flags as flags
     import modules.config
     import modules.patch
-    import fcbh.model_management
-    import fooocus_extras.preprocessors as preprocessors
+    import ldm_patched.modules.model_management
+    import extras.preprocessors as preprocessors
     import modules.inpaint_worker as inpaint_worker
     import modules.constants as constants
     import modules.advanced_parameters as advanced_parameters
-    import fooocus_extras.ip_adapter as ip_adapter
-    import fooocus_extras.face_crop
+    import extras.ip_adapter as ip_adapter
+    import extras.face_crop
 
     from modules.censor import censor_batch
 
     from modules.sdxl_styles import apply_style, apply_wildcards, fooocus_expansion
     from modules.private_logger import log
-    from modules.expansion import safe_str
+    from extras.expansion import safe_str
     from modules.util import remove_empty_str, HWC3, resize_image, \
         get_image_shape_ceil, set_image_shape_ceil, get_shape_ceil, resample_image
     from modules.upscaler import perform_upscale
@@ -665,7 +665,7 @@ def worker():
                 cn_img = HWC3(cn_img)
 
                 if not advanced_parameters.skipping_cn_preprocessor:
-                    cn_img = fooocus_extras.face_crop.crop_image(cn_img)
+                    cn_img = extras.face_crop.crop_image(cn_img)
 
                 # https://github.com/tencent-ailab/IP-Adapter/blob/d580c50a291566bbf9fc7ac0f760506607297e6d/README.md?plain=1#L75
                 cn_img = resize_image(cn_img, width=224, height=224, resize_mode=0)
@@ -798,7 +798,7 @@ def worker():
                     log(x, d, single_line_number=3)
 
                 yield_result(async_task, imgs, do_not_show_finished_images=len(tasks) == 1, progressbar_index=int(15.0 + 85.0 * float((current_task_id + 1) * steps) / float(all_steps)))
-            except fcbh.model_management.InterruptProcessingException as e:
+            except ldm_patched.modules.model_management.InterruptProcessingException as e:
                 if async_task.last_stop == 'skip':
                     print('User skipped')
                     async_task.last_stop = False

modules/core.py

@@ -8,26 +8,25 @@ import einops
 import torch
 import numpy as np
 
-import fcbh.model_management
-import fcbh.model_detection
-import fcbh.model_patcher
-import fcbh.utils
-import fcbh.controlnet
+import ldm_patched.modules.model_management
+import ldm_patched.modules.model_detection
+import ldm_patched.modules.model_patcher
+import ldm_patched.modules.utils
+import ldm_patched.modules.controlnet
 import modules.sample_hijack
-import fcbh.samplers
-import fcbh.latent_formats
+import ldm_patched.modules.samplers
+import ldm_patched.modules.latent_formats
 import modules.advanced_parameters
 
-from fcbh.sd import load_checkpoint_guess_config
-from nodes import VAEDecode, EmptyLatentImage, VAEEncode, VAEEncodeTiled, VAEDecodeTiled, \
+from ldm_patched.modules.sd import load_checkpoint_guess_config
+from ldm_patched.contrib.external import VAEDecode, EmptyLatentImage, VAEEncode, VAEEncodeTiled, VAEDecodeTiled, \
     ControlNetApplyAdvanced
-from fcbh_extras.nodes_freelunch import FreeU_V2
-from fcbh.sample import prepare_mask
-from modules.patch import patched_sampler_cfg_function
+from ldm_patched.contrib.external_freelunch import FreeU_V2
+from ldm_patched.modules.sample import prepare_mask
 from modules.lora import match_lora
-from fcbh.lora import model_lora_keys_unet, model_lora_keys_clip
+from ldm_patched.modules.lora import model_lora_keys_unet, model_lora_keys_clip
 from modules.config import path_embeddings
-from fcbh_extras.nodes_model_advanced import ModelSamplingDiscrete
+from ldm_patched.contrib.external_model_advanced import ModelSamplingDiscrete
 
 
 opEmptyLatentImage = EmptyLatentImage()
@@ -98,7 +97,7 @@ class StableDiffusionModel:
         self.clip_with_lora = self.clip.clone() if self.clip is not None else None
 
         for lora_filename, weight in loras_to_load:
-            lora_unmatch = fcbh.utils.load_torch_file(lora_filename, safe_load=False)
+            lora_unmatch = ldm_patched.modules.utils.load_torch_file(lora_filename, safe_load=False)
             lora_unet, lora_unmatch = match_lora(lora_unmatch, self.lora_key_map_unet)
             lora_clip, lora_unmatch = match_lora(lora_unmatch, self.lora_key_map_clip)
 
@@ -136,7 +135,7 @@ def apply_freeu(model, b1, b2, s1, s2):
 @torch.no_grad()
 @torch.inference_mode()
 def load_controlnet(ckpt_filename):
-    return fcbh.controlnet.load_controlnet(ckpt_filename)
+    return ldm_patched.modules.controlnet.load_controlnet(ckpt_filename)
 
 
 @torch.no_grad()
@@ -150,7 +149,6 @@ def apply_controlnet(positive, negative, control_net, image, strength, start_per
 @torch.inference_mode()
 def load_model(ckpt_filename):
     unet, clip, vae, clip_vision = load_checkpoint_guess_config(ckpt_filename, embedding_directory=path_embeddings)
-    unet.model_options['sampler_cfg_function'] = patched_sampler_cfg_function
     return StableDiffusionModel(unet=unet, clip=clip, vae=vae, clip_vision=clip_vision, filename=ckpt_filename)
 
 
@@ -230,7 +228,7 @@ def get_previewer(model):
     global VAE_approx_models
 
     from modules.config import path_vae_approx
-    is_sdxl = isinstance(model.model.latent_format, fcbh.latent_formats.SDXL)
+    is_sdxl = isinstance(model.model.latent_format, ldm_patched.modules.latent_formats.SDXL)
     vae_approx_filename = os.path.join(path_vae_approx, 'xlvaeapp.pth' if is_sdxl else 'vaeapp_sd15.pth')
 
     if vae_approx_filename in VAE_approx_models:
@@ -242,14 +240,14 @@ def get_previewer(model):
         del sd
         VAE_approx_model.eval()
 
-        if fcbh.model_management.should_use_fp16():
+        if ldm_patched.modules.model_management.should_use_fp16():
             VAE_approx_model.half()
             VAE_approx_model.current_type = torch.float16
         else:
             VAE_approx_model.float()
             VAE_approx_model.current_type = torch.float32
 
-        VAE_approx_model.to(fcbh.model_management.get_torch_device())
+        VAE_approx_model.to(ldm_patched.modules.model_management.get_torch_device())
         VAE_approx_models[vae_approx_filename] = VAE_approx_model
 
     @torch.no_grad()
@@ -273,7 +271,7 @@ def ksampler(model, positive, negative, latent, seed=None, steps=30, cfg=7.0, sa
              previewer_start=None, previewer_end=None, sigmas=None, noise_mean=None):
 
     if sigmas is not None:
-        sigmas = sigmas.clone().to(fcbh.model_management.get_torch_device())
+        sigmas = sigmas.clone().to(ldm_patched.modules.model_management.get_torch_device())
 
     latent_image = latent["samples"]
 
@@ -281,7 +279,7 @@ def ksampler(model, positive, negative, latent, seed=None, steps=30, cfg=7.0, sa
         noise = torch.zeros(latent_image.size(), dtype=latent_image.dtype, layout=latent_image.layout, device="cpu")
     else:
         batch_inds = latent["batch_index"] if "batch_index" in latent else None
-        noise = fcbh.sample.prepare_noise(latent_image, seed, batch_inds)
+        noise = ldm_patched.modules.sample.prepare_noise(latent_image, seed, batch_inds)
 
     if isinstance(noise_mean, torch.Tensor):
         noise = noise + noise_mean - torch.mean(noise, dim=1, keepdim=True)
@@ -299,7 +297,7 @@ def ksampler(model, positive, negative, latent, seed=None, steps=30, cfg=7.0, sa
         previewer_end = steps
 
     def callback(step, x0, x, total_steps):
-        fcbh.model_management.throw_exception_if_processing_interrupted()
+        ldm_patched.modules.model_management.throw_exception_if_processing_interrupted()
         y = None
         if previewer is not None and not modules.advanced_parameters.disable_preview and not (modules.config.default_black_out_nsfw and modules.config.default_hide_preview_if_black_out_nsfw):
             y = previewer(x0, previewer_start + step, previewer_end)
@@ -309,14 +307,18 @@ def ksampler(model, positive, negative, latent, seed=None, steps=30, cfg=7.0, sa
     disable_pbar = False
     modules.sample_hijack.current_refiner = refiner
     modules.sample_hijack.refiner_switch_step = refiner_switch
-    fcbh.samplers.sample = modules.sample_hijack.sample_hacked
+    ldm_patched.modules.samplers.sample = modules.sample_hijack.sample_hacked
 
     try:
-        samples = fcbh.sample.sample(model, noise, steps, cfg, sampler_name, scheduler, positive, negative, latent_image,
-                                      denoise=denoise, disable_noise=disable_noise, start_step=start_step,
-                                      last_step=last_step,
-                                      force_full_denoise=force_full_denoise, noise_mask=noise_mask, callback=callback,
-                                      disable_pbar=disable_pbar, seed=seed, sigmas=sigmas)
+        samples = ldm_patched.modules.sample.sample(model,
+                                                    noise, steps, cfg, sampler_name, scheduler,
+                                                    positive, negative, latent_image,
+                                                    denoise=denoise, disable_noise=disable_noise,
+                                                    start_step=start_step,
+                                                    last_step=last_step,
+                                                    force_full_denoise=force_full_denoise, noise_mask=noise_mask,
+                                                    callback=callback,
+                                                    disable_pbar=disable_pbar, seed=seed, sigmas=sigmas)
 
         out = latent.copy()
         out["samples"] = samples

modules/default_pipeline.py

@@ -3,13 +3,13 @@ import os
 import torch
 import modules.patch
 import modules.config
-import fcbh.model_management
-import fcbh.latent_formats
+import ldm_patched.modules.model_management
+import ldm_patched.modules.latent_formats
 import modules.inpaint_worker
-import fooocus_extras.vae_interpose as vae_interpose
+import extras.vae_interpose as vae_interpose
+from extras.expansion import FooocusExpansion
 
-from fcbh.model_base import SDXL, SDXLRefiner
-from modules.expansion import FooocusExpansion
+from ldm_patched.modules.model_base import SDXL, SDXLRefiner
 from modules.sample_hijack import clip_separate
 
 
@@ -208,7 +208,7 @@ def prepare_text_encoder(async_call=True):
         # TODO: make sure that this is always called in an async way so that users cannot feel it.
         pass
     assert_model_integrity()
-    fcbh.model_management.load_models_gpu([final_clip.patcher, final_expansion.patcher])
+    ldm_patched.modules.model_management.load_models_gpu([final_clip.patcher, final_expansion.patcher])
     return
 
 
@@ -270,7 +270,7 @@ def vae_parse(latent):
 @torch.no_grad()
 @torch.inference_mode()
 def calculate_sigmas_all(sampler, model, scheduler, steps):
-    from fcbh.samplers import calculate_sigmas_scheduler
+    from ldm_patched.modules.samplers import calculate_sigmas_scheduler
 
     discard_penultimate_sigma = False
     if sampler in ['dpm_2', 'dpm_2_ancestral']:
@@ -352,7 +352,7 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height
     print(f'[Sampler] sigma_min = {sigma_min}, sigma_max = {sigma_max}')
 
     modules.patch.BrownianTreeNoiseSamplerPatched.global_init(
-        initial_latent['samples'].to(fcbh.model_management.get_torch_device()),
+        initial_latent['samples'].to(ldm_patched.modules.model_management.get_torch_device()),
         sigma_min, sigma_max, seed=image_seed, cpu=False)
 
     decoded_latent = None

modules/expansion.py

@@ -1,126 +0,0 @@
-# Fooocus GPT2 Expansion
-# Algorithm created by Lvmin Zhang at 2023, Stanford
-# If used inside Fooocus, any use is permitted.
-# If used outside Fooocus, only non-commercial use is permitted (CC-By NC 4.0).
-# This applies to the word list, vocab, model, and algorithm.
-
-
-import os
-import torch
-import math
-import fcbh.model_management as model_management
-
-from transformers.generation.logits_process import LogitsProcessorList
-from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed
-from modules.config import path_fooocus_expansion
-from fcbh.model_patcher import ModelPatcher
-
-
-# limitation of np.random.seed(), called from transformers.set_seed()
-SEED_LIMIT_NUMPY = 2**32
-neg_inf = - 8192.0
-
-
-def safe_str(x):
-    x = str(x)
-    for _ in range(16):
-        x = x.replace('  ', ' ')
-    return x.strip(",. \r\n")
-
-
-def remove_pattern(x, pattern):
-    for p in pattern:
-        x = x.replace(p, '')
-    return x
-
-
-class FooocusExpansion:
-    def __init__(self):
-        self.tokenizer = AutoTokenizer.from_pretrained(path_fooocus_expansion)
-
-        positive_words = open(os.path.join(path_fooocus_expansion, 'positive.txt'),
-                              encoding='utf-8').read().splitlines()
-        positive_words = ['Ġ' + x.lower() for x in positive_words if x != '']
-
-        self.logits_bias = torch.zeros((1, len(self.tokenizer.vocab)), dtype=torch.float32) + neg_inf
-
-        debug_list = []
-        for k, v in self.tokenizer.vocab.items():
-            if k in positive_words:
-                self.logits_bias[0, v] = 0
-                debug_list.append(k[1:])
-
-        print(f'Fooocus V2 Expansion: Vocab with {len(debug_list)} words.')
-
-        # debug_list = '\n'.join(sorted(debug_list))
-        # print(debug_list)
-
-        # t11 = self.tokenizer(',', return_tensors="np")
-        # t198 = self.tokenizer('\n', return_tensors="np")
-        # eos = self.tokenizer.eos_token_id
-
-        self.model = AutoModelForCausalLM.from_pretrained(path_fooocus_expansion)
-        self.model.eval()
-
-        load_device = model_management.text_encoder_device()
-        offload_device = model_management.text_encoder_offload_device()
-
-        # MPS hack
-        if model_management.is_device_mps(load_device):
-            load_device = torch.device('cpu')
-            offload_device = torch.device('cpu')
-
-        use_fp16 = model_management.should_use_fp16(device=load_device)
-
-        if use_fp16:
-            self.model.half()
-
-        self.patcher = ModelPatcher(self.model, load_device=load_device, offload_device=offload_device)
-        print(f'Fooocus Expansion engine loaded for {load_device}, use_fp16 = {use_fp16}.')
-
-    @torch.no_grad()
-    @torch.inference_mode()
-    def logits_processor(self, input_ids, scores):
-        assert scores.ndim == 2 and scores.shape[0] == 1
-        self.logits_bias = self.logits_bias.to(scores)
-
-        bias = self.logits_bias.clone()
-        bias[0, input_ids[0].to(bias.device).long()] = neg_inf
-        bias[0, 11] = 0
-
-        return scores + bias
-
-    @torch.no_grad()
-    @torch.inference_mode()
-    def __call__(self, prompt, seed):
-        if prompt == '':
-            return ''
-
-        if self.patcher.current_device != self.patcher.load_device:
-            print('Fooocus Expansion loaded by itself.')
-            model_management.load_model_gpu(self.patcher)
-
-        seed = int(seed) % SEED_LIMIT_NUMPY
-        set_seed(seed)
-        prompt = safe_str(prompt) + ','
-
-        tokenized_kwargs = self.tokenizer(prompt, return_tensors="pt")
-        tokenized_kwargs.data['input_ids'] = tokenized_kwargs.data['input_ids'].to(self.patcher.load_device)
-        tokenized_kwargs.data['attention_mask'] = tokenized_kwargs.data['attention_mask'].to(self.patcher.load_device)
-
-        current_token_length = int(tokenized_kwargs.data['input_ids'].shape[1])
-        max_token_length = 75 * int(math.ceil(float(current_token_length) / 75.0))
-        max_new_tokens = max_token_length - current_token_length
-
-        # https://huggingface.co/blog/introducing-csearch
-        # https://huggingface.co/docs/transformers/generation_strategies
-        features = self.model.generate(**tokenized_kwargs,
-                                       top_k=100,
-                                       max_new_tokens=max_new_tokens,
-                                       do_sample=True,
-                                       logits_processor=LogitsProcessorList([self.logits_processor]))
-
-        response = self.tokenizer.batch_decode(features, skip_special_tokens=True)
-        result = safe_str(response[0])
-
-        return result

modules/flags.py

@@ -14,7 +14,7 @@ KSAMPLER_NAMES = ["euler", "euler_ancestral", "heun", "heunpp2","dpm_2", "dpm_2_
                   "lms", "dpm_fast", "dpm_adaptive", "dpmpp_2s_ancestral", "dpmpp_sde", "dpmpp_sde_gpu",
                   "dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm", "lcm"]
 
-SCHEDULER_NAMES = ["normal", "karras", "exponential", "sgm_uniform", "simple", "ddim_uniform", "lcm"]
+SCHEDULER_NAMES = ["normal", "karras", "exponential", "sgm_uniform", "simple", "ddim_uniform", "lcm", "turbo"]
 SAMPLER_NAMES = KSAMPLER_NAMES + ["ddim", "uni_pc", "uni_pc_bh2"]
 
 sampler_list = SAMPLER_NAMES
@@ -44,3 +44,6 @@ inpaint_option_default = 'Inpaint or Outpaint (default)'
 inpaint_option_detail = 'Improve Detail (face, hand, eyes, etc.)'
 inpaint_option_modify = 'Modify Content (add objects, change background, etc.)'
 inpaint_options = [inpaint_option_default, inpaint_option_detail, inpaint_option_modify]
+
+desc_type_photo = 'Photograph'
+desc_type_anime = 'Art/Anime'

modules/lora.py

@@ -4,7 +4,7 @@ def match_lora(lora, to_load):
     for x in to_load:
         real_load_key = to_load[x]
         if real_load_key in lora:
-            patch_dict[real_load_key] = lora[real_load_key]
+            patch_dict[real_load_key] = ('fooocus', lora[real_load_key])
             loaded_keys.add(real_load_key)
             continue
 
@@ -37,7 +37,7 @@ def match_lora(lora, to_load):
             if mid_name is not None and mid_name in lora.keys():
                 mid = lora[mid_name]
                 loaded_keys.add(mid_name)
-            patch_dict[to_load[x]] = (lora[A_name], lora[B_name], alpha, mid)
+            patch_dict[to_load[x]] = ("lora", (lora[A_name], lora[B_name], alpha, mid))
             loaded_keys.add(A_name)
             loaded_keys.add(B_name)
 
@@ -58,7 +58,7 @@ def match_lora(lora, to_load):
                 loaded_keys.add(hada_t1_name)
                 loaded_keys.add(hada_t2_name)
 
-            patch_dict[to_load[x]] = (lora[hada_w1_a_name], lora[hada_w1_b_name], alpha, lora[hada_w2_a_name], lora[hada_w2_b_name], hada_t1, hada_t2)
+            patch_dict[to_load[x]] = ("loha", (lora[hada_w1_a_name], lora[hada_w1_b_name], alpha, lora[hada_w2_a_name], lora[hada_w2_b_name], hada_t1, hada_t2))
             loaded_keys.add(hada_w1_a_name)
             loaded_keys.add(hada_w1_b_name)
             loaded_keys.add(hada_w2_a_name)
@@ -110,7 +110,19 @@ def match_lora(lora, to_load):
             loaded_keys.add(lokr_t2_name)
 
         if (lokr_w1 is not None) or (lokr_w2 is not None) or (lokr_w1_a is not None) or (lokr_w2_a is not None):
-            patch_dict[to_load[x]] = (lokr_w1, lokr_w2, alpha, lokr_w1_a, lokr_w1_b, lokr_w2_a, lokr_w2_b, lokr_t2)
+            patch_dict[to_load[x]] = ("lokr", (lokr_w1, lokr_w2, alpha, lokr_w1_a, lokr_w1_b, lokr_w2_a, lokr_w2_b, lokr_t2))
+
+        #glora
+        a1_name = "{}.a1.weight".format(x)
+        a2_name = "{}.a2.weight".format(x)
+        b1_name = "{}.b1.weight".format(x)
+        b2_name = "{}.b2.weight".format(x)
+        if a1_name in lora:
+            patch_dict[to_load[x]] = ("glora", (lora[a1_name], lora[a2_name], lora[b1_name], lora[b2_name], alpha))
+            loaded_keys.add(a1_name)
+            loaded_keys.add(a2_name)
+            loaded_keys.add(b1_name)
+            loaded_keys.add(b2_name)
 
         w_norm_name = "{}.w_norm".format(x)
         b_norm_name = "{}.b_norm".format(x)
@@ -119,21 +131,21 @@ def match_lora(lora, to_load):
 
         if w_norm is not None:
             loaded_keys.add(w_norm_name)
-            patch_dict[to_load[x]] = (w_norm,)
+            patch_dict[to_load[x]] = ("diff", (w_norm,))
             if b_norm is not None:
                 loaded_keys.add(b_norm_name)
-                patch_dict["{}.bias".format(to_load[x][:-len(".weight")])] = (b_norm,)
+                patch_dict["{}.bias".format(to_load[x][:-len(".weight")])] = ("diff", (b_norm,))
 
         diff_name = "{}.diff".format(x)
         diff_weight = lora.get(diff_name, None)
         if diff_weight is not None:
-            patch_dict[to_load[x]] = (diff_weight,)
+            patch_dict[to_load[x]] = ("diff", (diff_weight,))
             loaded_keys.add(diff_name)
 
         diff_bias_name = "{}.diff_b".format(x)
         diff_bias = lora.get(diff_bias_name, None)
         if diff_bias is not None:
-            patch_dict["{}.bias".format(to_load[x][:-len(".weight")])] = (diff_bias,)
+            patch_dict["{}.bias".format(to_load[x][:-len(".weight")])] = ("diff", (diff_bias,))
             loaded_keys.add(diff_bias_name)
 
     remaining_dict = {x: y for x, y in lora.items() if x not in loaded_keys}

modules/patch.py

@@ -1,33 +1,30 @@
 import os
 import torch
-import math
 import time
-import numpy as np
-import fcbh.model_base
-import fcbh.ldm.modules.diffusionmodules.openaimodel
-import fcbh.samplers
-import fcbh.model_management
+import math
+import ldm_patched.modules.model_base
+import ldm_patched.ldm.modules.diffusionmodules.openaimodel
+import ldm_patched.modules.model_management
 import modules.anisotropic as anisotropic
-import fcbh.ldm.modules.attention
-import fcbh.k_diffusion.sampling
-import fcbh.sd1_clip
+import ldm_patched.ldm.modules.attention
+import ldm_patched.k_diffusion.sampling
+import ldm_patched.modules.sd1_clip
 import modules.inpaint_worker as inpaint_worker
-import fcbh.ldm.modules.diffusionmodules.openaimodel
-import fcbh.ldm.modules.diffusionmodules.model
-import fcbh.sd
-import fcbh.cldm.cldm
-import fcbh.model_patcher
-import fcbh.samplers
-import fcbh.cli_args
+import ldm_patched.ldm.modules.diffusionmodules.openaimodel
+import ldm_patched.ldm.modules.diffusionmodules.model
+import ldm_patched.modules.sd
+import ldm_patched.controlnet.cldm
+import ldm_patched.modules.model_patcher
+import ldm_patched.modules.samplers
+import ldm_patched.modules.args_parser
 import modules.advanced_parameters as advanced_parameters
 import warnings
 import safetensors.torch
 import modules.constants as constants
 
-from einops import repeat
-from fcbh.k_diffusion.sampling import BatchedBrownianTree
-from fcbh.ldm.modules.diffusionmodules.openaimodel import forward_timestep_embed, apply_control
-from fcbh.ldm.modules.diffusionmodules.util import make_beta_schedule
+from ldm_patched.modules.samplers import calc_cond_uncond_batch
+from ldm_patched.k_diffusion.sampling import BatchedBrownianTree
+from ldm_patched.ldm.modules.diffusionmodules.openaimodel import forward_timestep_embed, apply_control
 
 
 sharpness = 2.0
@@ -54,31 +51,25 @@ def calculate_weight_patched(self, patches, weight, key):
             v = (self.calculate_weight(v[1:], v[0].clone(), key),)
 
         if len(v) == 1:
+            patch_type = "diff"
+        elif len(v) == 2:
+            patch_type = v[0]
+            v = v[1]
+
+        if patch_type == "diff":
             w1 = v[0]
             if alpha != 0.0:
                 if w1.shape != weight.shape:
                     print("WARNING SHAPE MISMATCH {} WEIGHT NOT MERGED {} != {}".format(key, w1.shape, weight.shape))
                 else:
-                    weight += alpha * fcbh.model_management.cast_to_device(w1, weight.device, weight.dtype)
-        elif len(v) == 3:
-            # fooocus
-            w1 = fcbh.model_management.cast_to_device(v[0], weight.device, torch.float32)
-            w_min = fcbh.model_management.cast_to_device(v[1], weight.device, torch.float32)
-            w_max = fcbh.model_management.cast_to_device(v[2], weight.device, torch.float32)
-            w1 = (w1 / 255.0) * (w_max - w_min) + w_min
-            if alpha != 0.0:
-                if w1.shape != weight.shape:
-                    print("WARNING SHAPE MISMATCH {} FOOOCUS WEIGHT NOT MERGED {} != {}".format(key, w1.shape, weight.shape))
-                else:
-                    weight += alpha * fcbh.model_management.cast_to_device(w1, weight.device, weight.dtype)
-        elif len(v) == 4:  # lora/locon
-            mat1 = fcbh.model_management.cast_to_device(v[0], weight.device, torch.float32)
-            mat2 = fcbh.model_management.cast_to_device(v[1], weight.device, torch.float32)
+                    weight += alpha * ldm_patched.modules.model_management.cast_to_device(w1, weight.device, weight.dtype)
+        elif patch_type == "lora":
+            mat1 = ldm_patched.modules.model_management.cast_to_device(v[0], weight.device, torch.float32)
+            mat2 = ldm_patched.modules.model_management.cast_to_device(v[1], weight.device, torch.float32)
             if v[2] is not None:
                 alpha *= v[2] / mat2.shape[0]
             if v[3] is not None:
-                # locon mid weights, hopefully the math is fine because I didn't properly test it
-                mat3 = fcbh.model_management.cast_to_device(v[3], weight.device, torch.float32)
+                mat3 = ldm_patched.modules.model_management.cast_to_device(v[3], weight.device, torch.float32)
                 final_shape = [mat2.shape[1], mat2.shape[0], mat3.shape[2], mat3.shape[3]]
                 mat2 = torch.mm(mat2.transpose(0, 1).flatten(start_dim=1),
                                 mat3.transpose(0, 1).flatten(start_dim=1)).reshape(final_shape).transpose(0, 1)
@@ -87,7 +78,17 @@ def calculate_weight_patched(self, patches, weight, key):
                     weight.shape).type(weight.dtype)
             except Exception as e:
                 print("ERROR", key, e)
-        elif len(v) == 8:  # lokr
+        elif patch_type == "fooocus":
+            w1 = ldm_patched.modules.model_management.cast_to_device(v[0], weight.device, torch.float32)
+            w_min = ldm_patched.modules.model_management.cast_to_device(v[1], weight.device, torch.float32)
+            w_max = ldm_patched.modules.model_management.cast_to_device(v[2], weight.device, torch.float32)
+            w1 = (w1 / 255.0) * (w_max - w_min) + w_min
+            if alpha != 0.0:
+                if w1.shape != weight.shape:
+                    print("WARNING SHAPE MISMATCH {} FOOOCUS WEIGHT NOT MERGED {} != {}".format(key, w1.shape, weight.shape))
+                else:
+                    weight += alpha * ldm_patched.modules.model_management.cast_to_device(w1, weight.device, weight.dtype)
+        elif patch_type == "lokr":
             w1 = v[0]
             w2 = v[1]
             w1_a = v[3]
@@ -99,23 +100,23 @@ def calculate_weight_patched(self, patches, weight, key):
 
             if w1 is None:
                 dim = w1_b.shape[0]
-                w1 = torch.mm(fcbh.model_management.cast_to_device(w1_a, weight.device, torch.float32),
-                              fcbh.model_management.cast_to_device(w1_b, weight.device, torch.float32))
+                w1 = torch.mm(ldm_patched.modules.model_management.cast_to_device(w1_a, weight.device, torch.float32),
+                              ldm_patched.modules.model_management.cast_to_device(w1_b, weight.device, torch.float32))
             else:
-                w1 = fcbh.model_management.cast_to_device(w1, weight.device, torch.float32)
+                w1 = ldm_patched.modules.model_management.cast_to_device(w1, weight.device, torch.float32)
 
             if w2 is None:
                 dim = w2_b.shape[0]
                 if t2 is None:
-                    w2 = torch.mm(fcbh.model_management.cast_to_device(w2_a, weight.device, torch.float32),
-                                  fcbh.model_management.cast_to_device(w2_b, weight.device, torch.float32))
+                    w2 = torch.mm(ldm_patched.modules.model_management.cast_to_device(w2_a, weight.device, torch.float32),
+                                  ldm_patched.modules.model_management.cast_to_device(w2_b, weight.device, torch.float32))
                 else:
                     w2 = torch.einsum('i j k l, j r, i p -> p r k l',
-                                      fcbh.model_management.cast_to_device(t2, weight.device, torch.float32),
-                                      fcbh.model_management.cast_to_device(w2_b, weight.device, torch.float32),
-                                      fcbh.model_management.cast_to_device(w2_a, weight.device, torch.float32))
+                                      ldm_patched.modules.model_management.cast_to_device(t2, weight.device, torch.float32),
+                                      ldm_patched.modules.model_management.cast_to_device(w2_b, weight.device, torch.float32),
+                                      ldm_patched.modules.model_management.cast_to_device(w2_a, weight.device, torch.float32))
             else:
-                w2 = fcbh.model_management.cast_to_device(w2, weight.device, torch.float32)
+                w2 = ldm_patched.modules.model_management.cast_to_device(w2, weight.device, torch.float32)
 
             if len(w2.shape) == 4:
                 w1 = w1.unsqueeze(2).unsqueeze(2)
@@ -126,7 +127,7 @@ def calculate_weight_patched(self, patches, weight, key):
                 weight += alpha * torch.kron(w1, w2).reshape(weight.shape).type(weight.dtype)
             except Exception as e:
                 print("ERROR", key, e)
-        else:  # loha
+        elif patch_type == "loha":
             w1a = v[0]
             w1b = v[1]
             if v[2] is not None:
@@ -137,24 +138,36 @@ def calculate_weight_patched(self, patches, weight, key):
                 t1 = v[5]
                 t2 = v[6]
                 m1 = torch.einsum('i j k l, j r, i p -> p r k l',
-                                  fcbh.model_management.cast_to_device(t1, weight.device, torch.float32),
-                                  fcbh.model_management.cast_to_device(w1b, weight.device, torch.float32),
-                                  fcbh.model_management.cast_to_device(w1a, weight.device, torch.float32))
+                                  ldm_patched.modules.model_management.cast_to_device(t1, weight.device, torch.float32),
+                                  ldm_patched.modules.model_management.cast_to_device(w1b, weight.device, torch.float32),
+                                  ldm_patched.modules.model_management.cast_to_device(w1a, weight.device, torch.float32))
 
                 m2 = torch.einsum('i j k l, j r, i p -> p r k l',
-                                  fcbh.model_management.cast_to_device(t2, weight.device, torch.float32),
-                                  fcbh.model_management.cast_to_device(w2b, weight.device, torch.float32),
-                                  fcbh.model_management.cast_to_device(w2a, weight.device, torch.float32))
+                                  ldm_patched.modules.model_management.cast_to_device(t2, weight.device, torch.float32),
+                                  ldm_patched.modules.model_management.cast_to_device(w2b, weight.device, torch.float32),
+                                  ldm_patched.modules.model_management.cast_to_device(w2a, weight.device, torch.float32))
             else:
-                m1 = torch.mm(fcbh.model_management.cast_to_device(w1a, weight.device, torch.float32),
-                              fcbh.model_management.cast_to_device(w1b, weight.device, torch.float32))
-                m2 = torch.mm(fcbh.model_management.cast_to_device(w2a, weight.device, torch.float32),
-                              fcbh.model_management.cast_to_device(w2b, weight.device, torch.float32))
+                m1 = torch.mm(ldm_patched.modules.model_management.cast_to_device(w1a, weight.device, torch.float32),
+                              ldm_patched.modules.model_management.cast_to_device(w1b, weight.device, torch.float32))
+                m2 = torch.mm(ldm_patched.modules.model_management.cast_to_device(w2a, weight.device, torch.float32),
+                              ldm_patched.modules.model_management.cast_to_device(w2b, weight.device, torch.float32))
 
             try:
                 weight += (alpha * m1 * m2).reshape(weight.shape).type(weight.dtype)
             except Exception as e:
                 print("ERROR", key, e)
+        elif patch_type == "glora":
+            if v[4] is not None:
+                alpha *= v[4] / v[0].shape[0]
+
+            a1 = ldm_patched.modules.model_management.cast_to_device(v[0].flatten(start_dim=1), weight.device, torch.float32)
+            a2 = ldm_patched.modules.model_management.cast_to_device(v[1].flatten(start_dim=1), weight.device, torch.float32)
+            b1 = ldm_patched.modules.model_management.cast_to_device(v[2].flatten(start_dim=1), weight.device, torch.float32)
+            b2 = ldm_patched.modules.model_management.cast_to_device(v[3].flatten(start_dim=1), weight.device, torch.float32)
+
+            weight += ((torch.mm(b2, b1) + torch.mm(torch.mm(weight.flatten(start_dim=1), a2), a1)) * alpha).reshape(weight.shape).type(weight.dtype)
+        else:
+            print("patch type not recognized", patch_type, key)
 
     return weight
 
@@ -162,19 +175,17 @@ def calculate_weight_patched(self, patches, weight, key):
 class BrownianTreeNoiseSamplerPatched:
     transform = None
     tree = None
-    global_sigma_min = 1.0
-    global_sigma_max = 1.0
 
     @staticmethod
     def global_init(x, sigma_min, sigma_max, seed=None, transform=lambda x: x, cpu=False):
+        if ldm_patched.modules.model_management.directml_enabled:
+            cpu = True
+
         t0, t1 = transform(torch.as_tensor(sigma_min)), transform(torch.as_tensor(sigma_max))
 
         BrownianTreeNoiseSamplerPatched.transform = transform
         BrownianTreeNoiseSamplerPatched.tree = BatchedBrownianTree(x, t0, t1, seed, cpu=cpu)
 
-        BrownianTreeNoiseSamplerPatched.global_sigma_min = sigma_min
-        BrownianTreeNoiseSamplerPatched.global_sigma_max = sigma_max
-
     def __init__(self, *args, **kwargs):
         pass
 
@@ -202,34 +213,47 @@ def compute_cfg(uncond, cond, cfg_scale, t):
         return real_eps
 
 
-def patched_sampler_cfg_function(args):
+def patched_sampling_function(model, x, timestep, uncond, cond, cond_scale, model_options=None, seed=None):
+    if math.isclose(cond_scale, 1.0):
+        return calc_cond_uncond_batch(model, cond, None, x, timestep, model_options)[0]
+
     global eps_record
 
-    positive_eps = args['cond']
-    negative_eps = args['uncond']
-    cfg_scale = args['cond_scale']
-    positive_x0 = args['input'] - positive_eps
-    sigma = args['sigma']
+    positive_x0, negative_x0 = calc_cond_uncond_batch(model, cond, uncond, x, timestep, model_options)
+
+    positive_eps = x - positive_x0
+    negative_eps = x - negative_x0
+    sigma = timestep
 
     alpha = 0.001 * sharpness * global_diffusion_progress
+
     positive_eps_degraded = anisotropic.adaptive_anisotropic_filter(x=positive_eps, g=positive_x0)
     positive_eps_degraded_weighted = positive_eps_degraded * alpha + positive_eps * (1.0 - alpha)
 
     final_eps = compute_cfg(uncond=negative_eps, cond=positive_eps_degraded_weighted,
-                            cfg_scale=cfg_scale, t=global_diffusion_progress)
+                            cfg_scale=cond_scale, t=global_diffusion_progress)
 
     if eps_record is not None:
         eps_record = (final_eps / sigma).cpu()
 
-    return final_eps
+    return x - final_eps
+
+
+def round_to_64(x):
+    h = float(x)
+    h = h / 64.0
+    h = round(h)
+    h = int(h)
+    h = h * 64
+    return h
 
 
 def sdxl_encode_adm_patched(self, **kwargs):
     global positive_adm_scale, negative_adm_scale
 
-    clip_pooled = fcbh.model_base.sdxl_pooled(kwargs, self.noise_augmentor)
-    width = kwargs.get("width", 768)
-    height = kwargs.get("height", 768)
+    clip_pooled = ldm_patched.modules.model_base.sdxl_pooled(kwargs, self.noise_augmentor)
+    width = kwargs.get("width", 1024)
+    height = kwargs.get("height", 1024)
     target_width = width
     target_height = height
 
@@ -240,25 +264,21 @@ def sdxl_encode_adm_patched(self, **kwargs):
         width = float(width) * positive_adm_scale
         height = float(height) * positive_adm_scale
 
-    # Avoid artifacts
-    width = int(width)
-    height = int(height)
-    crop_w = 0
-    crop_h = 0
-    target_width = int(target_width)
-    target_height = int(target_height)
+    def embedder(number_list):
+        h = [self.embedder(torch.Tensor([number])) for number in number_list]
+        y = torch.flatten(torch.cat(h)).unsqueeze(dim=0).repeat(clip_pooled.shape[0], 1)
+        return y
 
-    out_a = [self.embedder(torch.Tensor([height])), self.embedder(torch.Tensor([width])),
-             self.embedder(torch.Tensor([crop_h])), self.embedder(torch.Tensor([crop_w])),
-             self.embedder(torch.Tensor([target_height])), self.embedder(torch.Tensor([target_width]))]
-    flat_a = torch.flatten(torch.cat(out_a)).unsqueeze(dim=0).repeat(clip_pooled.shape[0], 1)
+    width, height = round_to_64(width), round_to_64(height)
+    target_width, target_height = round_to_64(target_width), round_to_64(target_height)
 
-    out_b = [self.embedder(torch.Tensor([target_height])), self.embedder(torch.Tensor([target_width])),
-             self.embedder(torch.Tensor([crop_h])), self.embedder(torch.Tensor([crop_w])),
-             self.embedder(torch.Tensor([target_height])), self.embedder(torch.Tensor([target_width]))]
-    flat_b = torch.flatten(torch.cat(out_b)).unsqueeze(dim=0).repeat(clip_pooled.shape[0], 1)
+    adm_emphasized = embedder([height, width, 0, 0, target_height, target_width])
+    adm_consistent = embedder([target_height, target_width, 0, 0, target_height, target_width])
 
-    return torch.cat((clip_pooled.to(flat_a.device), flat_a, clip_pooled.to(flat_b.device), flat_b), dim=1)
+    clip_pooled = clip_pooled.to(adm_emphasized)
+    final_adm = torch.cat((clip_pooled, adm_emphasized, clip_pooled, adm_consistent), dim=1)
+
+    return final_adm
 
 
 def encode_token_weights_patched_with_a1111_method(self, token_weight_pairs):
@@ -273,11 +293,11 @@ def encode_token_weights_patched_with_a1111_method(self, token_weight_pairs):
 
     sections = len(to_encode)
     if has_weights or sections == 0:
-        to_encode.append(fcbh.sd1_clip.gen_empty_tokens(self.special_tokens, max_token_len))
+        to_encode.append(ldm_patched.modules.sd1_clip.gen_empty_tokens(self.special_tokens, max_token_len))
 
     out, pooled = self.encode(to_encode)
     if pooled is not None:
-        first_pooled = pooled[0:1].cpu()
+        first_pooled = pooled[0:1].to(ldm_patched.modules.model_management.intermediate_device())
     else:
         first_pooled = pooled
 
@@ -297,9 +317,8 @@ def encode_token_weights_patched_with_a1111_method(self, token_weight_pairs):
         output.append(z)
 
     if len(output) == 0:
-        return out[-1:].cpu(), first_pooled
-
-    return torch.cat(output, dim=-2).cpu(), first_pooled
+        return out[-1:].to(ldm_patched.modules.model_management.intermediate_device()), first_pooled
+    return torch.cat(output, dim=-2).to(ldm_patched.modules.model_management.intermediate_device()), first_pooled
 
 
 def patched_KSamplerX0Inpaint_forward(self, x, sigma, uncond, cond, cond_scale, denoise_mask, model_options={}, seed=None):
@@ -344,27 +363,8 @@ def timed_adm(y, timesteps):
     return y
 
 
-def patched_timestep_embedding(timesteps, dim, max_period=10000, repeat_only=False):
-    # Consistent with Kohya to reduce differences between model training and inference.
-
-    if not repeat_only:
-        half = dim // 2
-        freqs = torch.exp(
-            -math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half
-        ).to(device=timesteps.device)
-        args = timesteps[:, None].float() * freqs[None]
-        embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
-        if dim % 2:
-            embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
-    else:
-        embedding = repeat(timesteps, 'b -> b d', d=dim)
-    return embedding
-
-
 def patched_cldm_forward(self, x, hint, timesteps, context, y=None, **kwargs):
-    t_emb = fcbh.ldm.modules.diffusionmodules.openaimodel.timestep_embedding(
-        timesteps, self.model_channels, repeat_only=False).to(self.dtype)
-
+    t_emb = ldm_patched.ldm.modules.diffusionmodules.openaimodel.timestep_embedding(timesteps, self.model_channels, repeat_only=False).to(x.dtype)
     emb = self.time_embed(t_emb)
 
     guided_hint = self.input_hint_block(hint, emb, context)
@@ -378,7 +378,7 @@ def patched_cldm_forward(self, x, hint, timesteps, context, y=None, **kwargs):
         assert y.shape[0] == x.shape[0]
         emb = emb + self.label_emb(y)
 
-    h = x.type(self.dtype)
+    h = x
     for module, zero_conv in zip(self.input_blocks, self.zero_convs):
         if guided_hint is not None:
             h = module(h, emb, context)
@@ -405,25 +405,31 @@ def patched_unet_forward(self, x, timesteps=None, context=None, y=None, control=
     self.current_step = 1.0 - timesteps.to(x) / 999.0
     global_diffusion_progress = float(self.current_step.detach().cpu().numpy().tolist()[0])
 
-    transformer_options["original_shape"] = list(x.shape)
-    transformer_options["current_index"] = 0
-    transformer_patches = transformer_options.get("patches", {})
-
     y = timed_adm(y, timesteps)
 
+    transformer_options["original_shape"] = list(x.shape)
+    transformer_options["transformer_index"] = 0
+    transformer_patches = transformer_options.get("patches", {})
+
+    num_video_frames = kwargs.get("num_video_frames", self.default_num_video_frames)
+    image_only_indicator = kwargs.get("image_only_indicator", self.default_image_only_indicator)
+    time_context = kwargs.get("time_context", None)
+
+    assert (y is not None) == (
+            self.num_classes is not None
+    ), "must specify y if and only if the model is class-conditional"
     hs = []
-    t_emb = fcbh.ldm.modules.diffusionmodules.openaimodel.timestep_embedding(
-        timesteps, self.model_channels, repeat_only=False).to(self.dtype)
+    t_emb = ldm_patched.ldm.modules.diffusionmodules.openaimodel.timestep_embedding(timesteps, self.model_channels, repeat_only=False).to(x.dtype)
     emb = self.time_embed(t_emb)
 
     if self.num_classes is not None:
         assert y.shape[0] == x.shape[0]
         emb = emb + self.label_emb(y)
 
-    h = x.type(self.dtype)
+    h = x
     for id, module in enumerate(self.input_blocks):
         transformer_options["block"] = ("input", id)
-        h = forward_timestep_embed(module, h, emb, context, transformer_options)
+        h = forward_timestep_embed(module, h, emb, context, transformer_options, time_context=time_context, num_video_frames=num_video_frames, image_only_indicator=image_only_indicator)
         h = apply_control(h, control, 'input')
         if "input_block_patch" in transformer_patches:
             patch = transformer_patches["input_block_patch"]
@@ -437,7 +443,7 @@ def patched_unet_forward(self, x, timesteps=None, context=None, y=None, control=
                 h = p(h, transformer_options)
 
     transformer_options["block"] = ("middle", 0)
-    h = forward_timestep_embed(self.middle_block, h, emb, context, transformer_options)
+    h = forward_timestep_embed(self.middle_block, h, emb, context, transformer_options, time_context=time_context, num_video_frames=num_video_frames, image_only_indicator=image_only_indicator)
     h = apply_control(h, control, 'middle')
 
     for id, module in enumerate(self.output_blocks):
@@ -456,7 +462,7 @@ def patched_unet_forward(self, x, timesteps=None, context=None, y=None, control=
             output_shape = hs[-1].shape
         else:
             output_shape = None
-        h = forward_timestep_embed(module, h, emb, context, transformer_options, output_shape)
+        h = forward_timestep_embed(module, h, emb, context, transformer_options, output_shape, time_context=time_context, num_video_frames=num_video_frames, image_only_indicator=image_only_indicator)
     h = h.type(x.dtype)
     if self.predict_codebook_ids:
         return self.id_predictor(h)
@@ -464,34 +470,9 @@ def patched_unet_forward(self, x, timesteps=None, context=None, y=None, control=
         return self.out(h)
 
 
-def patched_register_schedule(self, given_betas=None, beta_schedule="linear", timesteps=1000,
-                          linear_start=1e-4, linear_end=2e-2, cosine_s=8e-3):
-    # Consistent with Kohya to reduce differences between model training and inference.
-
-    if given_betas is not None:
-        betas = given_betas
-    else:
-        betas = make_beta_schedule(
-            beta_schedule,
-            timesteps,
-            linear_start=linear_start,
-            linear_end=linear_end,
-            cosine_s=cosine_s)
-
-    alphas = 1. - betas
-    alphas_cumprod = np.cumprod(alphas, axis=0)
-    timesteps, = betas.shape
-    self.num_timesteps = int(timesteps)
-    self.linear_start = linear_start
-    self.linear_end = linear_end
-    sigmas = torch.tensor(((1 - alphas_cumprod) / alphas_cumprod) ** 0.5, dtype=torch.float32)
-    self.set_sigmas(sigmas)
-    return
-
-
 def patched_load_models_gpu(*args, **kwargs):
     execution_start_time = time.perf_counter()
-    y = fcbh.model_management.load_models_gpu_origin(*args, **kwargs)
+    y = ldm_patched.modules.model_management.load_models_gpu_origin(*args, **kwargs)
     moving_time = time.perf_counter() - execution_start_time
     if moving_time > 0.1:
         print(f'[Fooocus Model Management] Moving model(s) has taken {moving_time:.2f} seconds')
@@ -533,19 +514,18 @@ def build_loaded(module, loader_name):
 
 
 def patch_all():
-    if not hasattr(fcbh.model_management, 'load_models_gpu_origin'):
-        fcbh.model_management.load_models_gpu_origin = fcbh.model_management.load_models_gpu
+    if not hasattr(ldm_patched.modules.model_management, 'load_models_gpu_origin'):
+        ldm_patched.modules.model_management.load_models_gpu_origin = ldm_patched.modules.model_management.load_models_gpu
 
-    fcbh.model_management.load_models_gpu = patched_load_models_gpu
-    fcbh.model_patcher.ModelPatcher.calculate_weight = calculate_weight_patched
-    fcbh.cldm.cldm.ControlNet.forward = patched_cldm_forward
-    fcbh.ldm.modules.diffusionmodules.openaimodel.UNetModel.forward = patched_unet_forward
-    fcbh.model_base.SDXL.encode_adm = sdxl_encode_adm_patched
-    fcbh.sd1_clip.ClipTokenWeightEncoder.encode_token_weights = encode_token_weights_patched_with_a1111_method
-    fcbh.samplers.KSamplerX0Inpaint.forward = patched_KSamplerX0Inpaint_forward
-    fcbh.k_diffusion.sampling.BrownianTreeNoiseSampler = BrownianTreeNoiseSamplerPatched
-    fcbh.ldm.modules.diffusionmodules.openaimodel.timestep_embedding = patched_timestep_embedding
-    fcbh.model_base.ModelSamplingDiscrete._register_schedule = patched_register_schedule
+    ldm_patched.modules.model_management.load_models_gpu = patched_load_models_gpu
+    ldm_patched.modules.model_patcher.ModelPatcher.calculate_weight = calculate_weight_patched
+    ldm_patched.controlnet.cldm.ControlNet.forward = patched_cldm_forward
+    ldm_patched.ldm.modules.diffusionmodules.openaimodel.UNetModel.forward = patched_unet_forward
+    ldm_patched.modules.model_base.SDXL.encode_adm = sdxl_encode_adm_patched
+    ldm_patched.modules.sd1_clip.ClipTokenWeightEncoder.encode_token_weights = encode_token_weights_patched_with_a1111_method
+    ldm_patched.modules.samplers.KSamplerX0Inpaint.forward = patched_KSamplerX0Inpaint_forward
+    ldm_patched.k_diffusion.sampling.BrownianTreeNoiseSampler = BrownianTreeNoiseSamplerPatched
+    ldm_patched.modules.samplers.sampling_function = patched_sampling_function
 
     warnings.filterwarnings(action='ignore', module='torchsde')
 

modules/private_logger.py

@@ -35,16 +35,19 @@ def log(img, dic, single_line_number=3):
 
     div_name = only_name.replace('.', '_')
     item = f'<div id="{div_name}">\n'
-    item += f"<p>{only_name}</p>\n"
+    item += "<table><tr>"
+    item += f"<td><img src=\"{only_name}\" width=auto height=100% loading=lazy style=\"height:auto;max-width:512px\" onerror=\"document.getElementById('{div_name}').style.display = 'none';\"></img></p></td>"
+    item += f"<td style=\"padding-left:10px;\"><p>{only_name}</p>\n"
     for i, (k, v) in enumerate(dic):
         if i < single_line_number:
-            item += f"<p>{k}: <b>{v}</b> </p>\n"
+            item += f"<p>{k}: <b>{v}</b></p>\n"
         else:
             if (i - single_line_number) % 2 == 0:
                 item += f"<p>{k}: <b>{v}</b>, "
             else:
                 item += f"{k}: <b>{v}</b></p>\n"
-    item += f"<p><img src=\"{only_name}\" width=auto height=100% loading=lazy style=\"height:auto;max-width:512px\" onerror=\"document.getElementById('{div_name}').style.display = 'none';\"></img></p><hr></div>\n"
+    item += "</td>"
+    item += "</tr></table><hr></div>\n"
     existing_log = item + existing_log
 
     with open(html_name, 'w', encoding='utf-8') as f:

modules/sample_hijack.py

@@ -1,11 +1,15 @@
 import torch
-import fcbh.samplers
-import fcbh.model_management
+import ldm_patched.modules.samplers
+import ldm_patched.modules.model_management
 
-from fcbh.model_base import SDXLRefiner, SDXL
-from fcbh.conds import CONDRegular
-from fcbh.sample import get_additional_models, get_models_from_cond, cleanup_additional_models
-from fcbh.samplers import resolve_areas_and_cond_masks, wrap_model, calculate_start_end_timesteps, \
+from collections import namedtuple
+from ldm_patched.contrib.external_custom_sampler import SDTurboScheduler
+from ldm_patched.k_diffusion import sampling as k_diffusion_sampling
+from ldm_patched.modules.samplers import normal_scheduler, simple_scheduler, ddim_scheduler
+from ldm_patched.modules.model_base import SDXLRefiner, SDXL
+from ldm_patched.modules.conds import CONDRegular
+from ldm_patched.modules.sample import get_additional_models, get_models_from_cond, cleanup_additional_models
+from ldm_patched.modules.samplers import resolve_areas_and_cond_masks, wrap_model, calculate_start_end_timesteps, \
     create_cond_with_same_area_if_none, pre_run_control, apply_empty_x_to_equal_area, encode_model_conds
 
 
@@ -133,7 +137,9 @@ def sample_hacked(model, noise, positive, negative, cfg, device, sampler, sigmas
         extra_args['model_options'] = {k: {} if k == 'transformer_options' else v for k, v in extra_args['model_options'].items()}
 
         models, inference_memory = get_additional_models(positive_refiner, negative_refiner, current_refiner.model_dtype())
-        fcbh.model_management.load_models_gpu([current_refiner] + models, current_refiner.memory_required(noise.shape) + inference_memory)
+        ldm_patched.modules.model_management.load_models_gpu(
+            [current_refiner] + models,
+            model.memory_required([noise.shape[0] * 2] + list(noise.shape[1:])) + inference_memory)
 
         model_wrap.inner_model = current_refiner.model
         print('Refiner Swapped')
@@ -152,4 +158,27 @@ def sample_hacked(model, noise, positive, negative, cfg, device, sampler, sigmas
     return model.process_latent_out(samples.to(torch.float32))
 
 
-fcbh.samplers.sample = sample_hacked
+@torch.no_grad()
+@torch.inference_mode()
+def calculate_sigmas_scheduler_hacked(model, scheduler_name, steps):
+    if scheduler_name == "karras":
+        sigmas = k_diffusion_sampling.get_sigmas_karras(n=steps, sigma_min=float(model.model_sampling.sigma_min), sigma_max=float(model.model_sampling.sigma_max))
+    elif scheduler_name == "exponential":
+        sigmas = k_diffusion_sampling.get_sigmas_exponential(n=steps, sigma_min=float(model.model_sampling.sigma_min), sigma_max=float(model.model_sampling.sigma_max))
+    elif scheduler_name == "normal":
+        sigmas = normal_scheduler(model, steps)
+    elif scheduler_name == "simple":
+        sigmas = simple_scheduler(model, steps)
+    elif scheduler_name == "ddim_uniform":
+        sigmas = ddim_scheduler(model, steps)
+    elif scheduler_name == "sgm_uniform":
+        sigmas = normal_scheduler(model, steps, sgm=True)
+    elif scheduler_name == "turbo":
+        sigmas = SDTurboScheduler().get_sigmas(namedtuple('Patcher', ['model'])(model=model), steps)[0]
+    else:
+        raise TypeError("error invalid scheduler")
+    return sigmas
+
+
+ldm_patched.modules.samplers.calculate_sigmas_scheduler = calculate_sigmas_scheduler_hacked
+ldm_patched.modules.samplers.sample = sample_hacked

modules/upscaler.py

@@ -2,8 +2,8 @@ import os
 import torch
 import modules.core as core
 
-from fcbh_extras.chainner_models.architecture.RRDB import RRDBNet as ESRGAN
-from fcbh_extras.nodes_upscale_model import ImageUpscaleWithModel
+from ldm_patched.pfn.architecture.RRDB import RRDBNet as ESRGAN
+from ldm_patched.contrib.external_upscale_model import ImageUpscaleWithModel
 from collections import OrderedDict
 from modules.config import path_upscale_models