mirror of
https://github.com/lllyasviel/Fooocus.git
synced 2026-08-16 13:13:16 +02:00
[Fooocus 2.0.60] Fooocus Inpaint or Outpaint (Midjourney Left/Right/Top/Bottom) (#402)
[Fooocus 2.0.60] Fooocus Inpaint or Outpaint (Midjourney Left/Right/Top/Bottom) (#402)
This commit is contained in:
lllyasviel
+93
-37
@@ -6,15 +6,20 @@ import comfy.k_diffusion.external
|
||||
import comfy.model_management
|
||||
import modules.anisotropic as anisotropic
|
||||
import comfy.ldm.modules.attention
|
||||
import comfy.k_diffusion.sampling
|
||||
import comfy.sd1_clip
|
||||
import modules.inpaint_worker as inpaint_worker
|
||||
|
||||
from comfy.k_diffusion import utils
|
||||
from comfy.k_diffusion.sampling import BrownianTreeNoiseSampler, trange
|
||||
|
||||
|
||||
sharpness = 2.0
|
||||
negative_adm = True
|
||||
|
||||
cfg_x0 = 0.0
|
||||
cfg_s = 1.0
|
||||
cfg_cin = 1.0
|
||||
|
||||
|
||||
def cfg_patched(args):
|
||||
@@ -37,14 +42,29 @@ def cfg_patched(args):
|
||||
|
||||
|
||||
def patched_discrete_eps_ddpm_denoiser_forward(self, input, sigma, **kwargs):
|
||||
global cfg_x0, cfg_s
|
||||
global cfg_x0, cfg_s, cfg_cin
|
||||
c_out, c_in = [utils.append_dims(x, input.ndim) for x in self.get_scalings(sigma)]
|
||||
cfg_x0 = input
|
||||
cfg_s = c_out
|
||||
cfg_cin = c_in
|
||||
return self.get_eps(input * c_in, self.sigma_to_t(sigma), **kwargs)
|
||||
|
||||
|
||||
def patched_model_function(func, args):
|
||||
global cfg_cin
|
||||
x = args['input']
|
||||
t = args['timestep']
|
||||
c = args['c']
|
||||
is_uncond = torch.tensor(args['cond_or_uncond'])[:, None, None, None].to(x) * 5e-3
|
||||
if inpaint_worker.current_task is not None:
|
||||
p = inpaint_worker.current_task.uc_guidance * cfg_cin
|
||||
x = p * is_uncond + x * (1 - is_uncond ** 2.0) ** 0.5
|
||||
return func(x, t, **c)
|
||||
|
||||
|
||||
def sdxl_encode_adm_patched(self, **kwargs):
|
||||
global negative_adm
|
||||
|
||||
clip_pooled = kwargs["pooled_output"]
|
||||
width = kwargs.get("width", 768)
|
||||
height = kwargs.get("height", 768)
|
||||
@@ -53,12 +73,13 @@ def sdxl_encode_adm_patched(self, **kwargs):
|
||||
target_width = kwargs.get("target_width", width)
|
||||
target_height = kwargs.get("target_height", height)
|
||||
|
||||
if kwargs.get("prompt_type", "") == "negative":
|
||||
width *= 0.8
|
||||
height *= 0.8
|
||||
elif kwargs.get("prompt_type", "") == "positive":
|
||||
width *= 1.5
|
||||
height *= 1.5
|
||||
if negative_adm:
|
||||
if kwargs.get("prompt_type", "") == "negative":
|
||||
width *= 0.8
|
||||
height *= 0.8
|
||||
elif kwargs.get("prompt_type", "") == "positive":
|
||||
width *= 1.5
|
||||
height *= 1.5
|
||||
|
||||
out = []
|
||||
out.append(self.embedder(torch.Tensor([height])))
|
||||
@@ -71,35 +92,6 @@ def sdxl_encode_adm_patched(self, **kwargs):
|
||||
return torch.cat((clip_pooled.to(flat.device), flat), dim=1)
|
||||
|
||||
|
||||
def sdxl_refiner_encode_adm_patched(self, **kwargs):
|
||||
clip_pooled = kwargs["pooled_output"]
|
||||
width = kwargs.get("width", 768)
|
||||
height = kwargs.get("height", 768)
|
||||
crop_w = kwargs.get("crop_w", 0)
|
||||
crop_h = kwargs.get("crop_h", 0)
|
||||
|
||||
if kwargs.get("prompt_type", "") == "negative":
|
||||
aesthetic_score = kwargs.get("aesthetic_score", 2.5)
|
||||
else:
|
||||
aesthetic_score = kwargs.get("aesthetic_score", 7.0)
|
||||
|
||||
if kwargs.get("prompt_type", "") == "negative":
|
||||
width *= 0.8
|
||||
height *= 0.8
|
||||
elif kwargs.get("prompt_type", "") == "positive":
|
||||
width *= 1.5
|
||||
height *= 1.5
|
||||
|
||||
out = []
|
||||
out.append(self.embedder(torch.Tensor([height])))
|
||||
out.append(self.embedder(torch.Tensor([width])))
|
||||
out.append(self.embedder(torch.Tensor([crop_h])))
|
||||
out.append(self.embedder(torch.Tensor([crop_w])))
|
||||
out.append(self.embedder(torch.Tensor([aesthetic_score])))
|
||||
flat = torch.flatten(torch.cat(out))[None,]
|
||||
return torch.cat((clip_pooled.to(flat.device), flat), dim=1)
|
||||
|
||||
|
||||
def text_encoder_device_patched():
|
||||
# Fooocus's style system uses text encoder much more times than comfy so this makes things much faster.
|
||||
return comfy.model_management.get_torch_device()
|
||||
@@ -138,15 +130,79 @@ def encode_token_weights_patched_with_a1111_method(self, token_weight_pairs):
|
||||
return torch.cat(output, dim=-2).cpu(), first_pooled.cpu()
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def sample_dpmpp_fooocus_2m_sde_inpaint_seamless(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, **kwargs):
|
||||
sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max()
|
||||
noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max, seed=extra_args.get("seed", None), cpu=False) if noise_sampler is None else noise_sampler
|
||||
|
||||
seed = extra_args.get("seed", None)
|
||||
assert isinstance(seed, int)
|
||||
|
||||
energy_generator = torch.Generator(device='cpu')
|
||||
energy_generator.manual_seed(seed + 1) # avoid bad results by using different seeds.
|
||||
|
||||
def get_energy():
|
||||
return torch.randn(x.size(), dtype=x.dtype, generator=energy_generator, device="cpu").to(x)
|
||||
|
||||
sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max()
|
||||
noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max, seed=seed, cpu=True) if noise_sampler is None else noise_sampler
|
||||
extra_args = {} if extra_args is None else extra_args
|
||||
s_in = x.new_ones([x.shape[0]])
|
||||
|
||||
old_denoised, h_last, h = None, None, None
|
||||
|
||||
latent_processor = model.inner_model.inner_model.inner_model.process_latent_in
|
||||
inpaint_latent = None
|
||||
inpaint_mask = None
|
||||
|
||||
if inpaint_worker.current_task is not None:
|
||||
inpaint_latent = latent_processor(inpaint_worker.current_task.latent).to(x)
|
||||
inpaint_mask = inpaint_worker.current_task.latent_mask.to(x)
|
||||
|
||||
def blend_latent(a, b, w):
|
||||
return a * w + b * (1 - w)
|
||||
|
||||
for i in trange(len(sigmas) - 1, disable=disable):
|
||||
if inpaint_latent is None:
|
||||
denoised = model(x, sigmas[i] * s_in, **extra_args)
|
||||
else:
|
||||
inpaint_worker.current_task.uc_guidance = x.detach().clone()
|
||||
energy = get_energy() * sigmas[i] + inpaint_latent
|
||||
x_prime = blend_latent(x, energy, inpaint_mask)
|
||||
denoised = model(x_prime, sigmas[i] * s_in, **extra_args)
|
||||
denoised = blend_latent(denoised, inpaint_latent, inpaint_mask)
|
||||
if callback is not None:
|
||||
callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised})
|
||||
if sigmas[i + 1] == 0:
|
||||
x = denoised
|
||||
else:
|
||||
t, s = -sigmas[i].log(), -sigmas[i + 1].log()
|
||||
h = s - t
|
||||
eta_h = eta * h
|
||||
|
||||
x = sigmas[i + 1] / sigmas[i] * (-eta_h).exp() * x + (-h - eta_h).expm1().neg() * denoised
|
||||
if old_denoised is not None:
|
||||
r = h_last / h
|
||||
x = x + 0.5 * (-h - eta_h).expm1().neg() * (1 / r) * (denoised - old_denoised)
|
||||
|
||||
x = x + noise_sampler(sigmas[i], sigmas[i + 1]) * sigmas[i + 1] * (
|
||||
-2 * eta_h).expm1().neg().sqrt() * s_noise
|
||||
|
||||
old_denoised = denoised
|
||||
h_last = h
|
||||
|
||||
return x
|
||||
|
||||
|
||||
def patch_all():
|
||||
comfy.ldm.modules.attention.print = lambda x: None
|
||||
comfy.k_diffusion.sampling.sample_dpmpp_fooocus_2m_sde_inpaint_seamless = sample_dpmpp_fooocus_2m_sde_inpaint_seamless
|
||||
|
||||
comfy.model_management.text_encoder_device = text_encoder_device_patched
|
||||
print(f'Fooocus Text Processing Pipelines are retargeted to {str(comfy.model_management.text_encoder_device())}')
|
||||
|
||||
comfy.k_diffusion.external.DiscreteEpsDDPMDenoiser.forward = patched_discrete_eps_ddpm_denoiser_forward
|
||||
comfy.model_base.SDXL.encode_adm = sdxl_encode_adm_patched
|
||||
# comfy.model_base.SDXLRefiner.encode_adm = sdxl_refiner_encode_adm_patched
|
||||
|
||||
comfy.sd1_clip.ClipTokenWeightEncoder.encode_token_weights = encode_token_weights_patched_with_a1111_method
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user