Foreign-Mirrors/Fooocus
1
0
Fork 0
mirror of https://github.com/lllyasviel/Fooocus.git synced 2026-08-16 13:13:16 +02:00
Code Issues Packages Projects Releases Wiki Activity

2.1.782

Browse Source 
2.1.782
This commit is contained in:
lllyasviel
2023-11-11 01:43:01 -08:00
parent a9bb1079cf
commit 4fe08161a5
48 changed files with 1041 additions and 2639 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/async_worker.py
+12 -10
View File
@@ -20,7 +20,7 @@ def worker():
import modules.default_pipeline as pipeline
import modules.core as core
import modules.flags as flags
import modules.path
import modules.config
import modules.patch
import fcbh.model_management
import fooocus_extras.preprocessors as preprocessors
@@ -143,7 +143,7 @@ def worker():
cn_tasks[cn_type].append([cn_img, cn_stop, cn_weight])
outpaint_selections = [o.lower() for o in outpaint_selections]
loras_raw = copy.deepcopy(loras)
base_model_additional_loras = []
raw_style_selections = copy.deepcopy(style_selections)
uov_method = uov_method.lower()
@@ -221,7 +221,7 @@ def worker():
else:
steps = 36
progressbar(1, 'Downloading upscale models ...')
modules.path.downloading_upscale_model()
modules.config.downloading_upscale_model()
if (current_tab == 'inpaint' or (current_tab == 'ip' and advanced_parameters.mixing_image_prompt_and_inpaint))\
and isinstance(inpaint_input_image, dict):
inpaint_image = inpaint_input_image['image']
@@ -230,8 +230,8 @@ def worker():
if isinstance(inpaint_image, np.ndarray) and isinstance(inpaint_mask, np.ndarray) \
and (np.any(inpaint_mask > 127) or len(outpaint_selections) > 0):
progressbar(1, 'Downloading inpainter ...')
inpaint_head_model_path, inpaint_patch_model_path = modules.path.downloading_inpaint_models(advanced_parameters.inpaint_engine)
loras += [(inpaint_patch_model_path, 1.0)]
inpaint_head_model_path, inpaint_patch_model_path = modules.config.downloading_inpaint_models(advanced_parameters.inpaint_engine)
base_model_additional_loras += [(inpaint_patch_model_path, 1.0)]
print(f'[Inpaint] Current inpaint model is {inpaint_patch_model_path}')
goals.append('inpaint')
if current_tab == 'ip' or \
@@ -240,11 +240,11 @@ def worker():
goals.append('cn')
progressbar(1, 'Downloading control models ...')
if len(cn_tasks[flags.cn_canny]) > 0:
controlnet_canny_path = modules.path.downloading_controlnet_canny()
controlnet_canny_path = modules.config.downloading_controlnet_canny()
if len(cn_tasks[flags.cn_cpds]) > 0:
controlnet_cpds_path = modules.path.downloading_controlnet_cpds()
controlnet_cpds_path = modules.config.downloading_controlnet_cpds()
if len(cn_tasks[flags.cn_ip]) > 0:
clip_vision_path, ip_negative_path, ip_adapter_path = modules.path.downloading_ip_adapters()
clip_vision_path, ip_negative_path, ip_adapter_path = modules.config.downloading_ip_adapters()
progressbar(1, 'Loading control models ...')
# Load or unload CNs
@@ -286,7 +286,8 @@ def worker():
extra_negative_prompts = negative_prompts[1:] if len(negative_prompts) > 1 else []
progressbar(3, 'Loading models ...')
pipeline.refresh_everything(refiner_model_name=refiner_model_name, base_model_name=base_model_name, loras=loras)
pipeline.refresh_everything(refiner_model_name=refiner_model_name, base_model_name=base_model_name,
loras=loras, base_model_additional_loras=base_model_additional_loras)
progressbar(3, 'Processing prompts ...')
tasks = []
@@ -618,11 +619,12 @@ def worker():
('ADM Guidance', str((modules.patch.positive_adm_scale, modules.patch.negative_adm_scale))),
('Base Model', base_model_name),
('Refiner Model', refiner_model_name),
('Refiner Switch', refiner_switch),
('Sampler', sampler_name),
('Scheduler', scheduler_name),
('Seed', task['task_seed'])
]
for n, w in loras_raw:
for n, w in loras:
if n != 'None':
d.append((f'LoRA [{n}] weight', w))
log(x, d, single_line_number=3)
modules/path.py → modules/config.py
+32 -33
View File
@@ -50,17 +50,16 @@ def get_dir_or_set_default(key, default_value):
return dp
modelfile_path = get_dir_or_set_default('modelfile_path', '../models/checkpoints/')
lorafile_path = get_dir_or_set_default('lorafile_path', '../models/loras/')
embeddings_path = get_dir_or_set_default('embeddings_path', '../models/embeddings/')
vae_approx_path = get_dir_or_set_default('vae_approx_path', '../models/vae_approx/')
upscale_models_path = get_dir_or_set_default('upscale_models_path', '../models/upscale_models/')
inpaint_models_path = get_dir_or_set_default('inpaint_models_path', '../models/inpaint/')
controlnet_models_path = get_dir_or_set_default('controlnet_models_path', '../models/controlnet/')
clip_vision_models_path = get_dir_or_set_default('clip_vision_models_path', '../models/clip_vision/')
fooocus_expansion_path = get_dir_or_set_default('fooocus_expansion_path',
'../models/prompt_expansion/fooocus_expansion')
temp_outputs_path = get_dir_or_set_default('temp_outputs_path', '../outputs/')
path_checkpoints = get_dir_or_set_default('modelfile_path', '../models/checkpoints/')
path_loras = get_dir_or_set_default('lorafile_path', '../models/loras/')
path_embeddings = get_dir_or_set_default('embeddings_path', '../models/embeddings/')
path_vae_approx = get_dir_or_set_default('vae_approx_path', '../models/vae_approx/')
path_upscale_models = get_dir_or_set_default('upscale_models_path', '../models/upscale_models/')
path_inpaint = get_dir_or_set_default('inpaint_models_path', '../models/inpaint/')
path_controlnet = get_dir_or_set_default('controlnet_models_path', '../models/controlnet/')
path_clip_vision = get_dir_or_set_default('clip_vision_models_path', '../models/clip_vision/')
path_fooocus_expansion = get_dir_or_set_default('fooocus_expansion_path', '../models/prompt_expansion/fooocus_expansion')
path_outputs = get_dir_or_set_default('temp_outputs_path', '../outputs/')
def get_config_item_or_set_default(key, default_value, validator, disable_empty_as_none=False):
@@ -93,7 +92,7 @@ default_refiner_model_name = get_config_item_or_set_default(
)
default_refiner_switch = get_config_item_or_set_default(
key='default_refiner_switch',
default_value=0.8,
default_value=0.5,
validator=lambda x: isinstance(x, float)
)
default_lora_name = get_config_item_or_set_default(
@@ -190,7 +189,7 @@ if preset is None:
with open(config_path, "w", encoding="utf-8") as json_file:
json.dump({k: config_dict[k] for k in visited_keys}, json_file, indent=4)
os.makedirs(temp_outputs_path, exist_ok=True)
os.makedirs(path_outputs, exist_ok=True)
model_filenames = []
lora_filenames = []
@@ -205,8 +204,8 @@ def get_model_filenames(folder_path, name_filter=None):
def update_all_model_names():
global model_filenames, lora_filenames
model_filenames = get_model_filenames(modelfile_path)
lora_filenames = get_model_filenames(lorafile_path)
model_filenames = get_model_filenames(path_checkpoints)
lora_filenames = get_model_filenames(path_loras)
return
@@ -215,10 +214,10 @@ def downloading_inpaint_models(v):
load_file_from_url(
url='https://huggingface.co/lllyasviel/fooocus_inpaint/resolve/main/fooocus_inpaint_head.pth',
model_dir=inpaint_models_path,
model_dir=path_inpaint,
file_name='fooocus_inpaint_head.pth'
)
head_file = os.path.join(inpaint_models_path, 'fooocus_inpaint_head.pth')
head_file = os.path.join(path_inpaint, 'fooocus_inpaint_head.pth')
patch_file = None
# load_file_from_url(
@@ -231,18 +230,18 @@ def downloading_inpaint_models(v):
if v == 'v1':
load_file_from_url(
url='https://huggingface.co/lllyasviel/fooocus_inpaint/resolve/main/inpaint.fooocus.patch',
model_dir=inpaint_models_path,
model_dir=path_inpaint,
file_name='inpaint.fooocus.patch'
)
patch_file = os.path.join(inpaint_models_path, 'inpaint.fooocus.patch')
patch_file = os.path.join(path_inpaint, 'inpaint.fooocus.patch')
if v == 'v2.5':
load_file_from_url(
url='https://huggingface.co/lllyasviel/fooocus_inpaint/resolve/main/inpaint_v25.fooocus.patch',
model_dir=inpaint_models_path,
model_dir=path_inpaint,
file_name='inpaint_v25.fooocus.patch'
)
patch_file = os.path.join(inpaint_models_path, 'inpaint_v25.fooocus.patch')
patch_file = os.path.join(path_inpaint, 'inpaint_v25.fooocus.patch')
return head_file, patch_file
@@ -250,19 +249,19 @@ def downloading_inpaint_models(v):
def downloading_controlnet_canny():
load_file_from_url(
url='https://huggingface.co/lllyasviel/misc/resolve/main/control-lora-canny-rank128.safetensors',
model_dir=controlnet_models_path,
model_dir=path_controlnet,
file_name='control-lora-canny-rank128.safetensors'
)
return os.path.join(controlnet_models_path, 'control-lora-canny-rank128.safetensors')
return os.path.join(path_controlnet, 'control-lora-canny-rank128.safetensors')
def downloading_controlnet_cpds():
load_file_from_url(
url='https://huggingface.co/lllyasviel/misc/resolve/main/fooocus_xl_cpds_128.safetensors',
model_dir=controlnet_models_path,
model_dir=path_controlnet,
file_name='fooocus_xl_cpds_128.safetensors'
)
return os.path.join(controlnet_models_path, 'fooocus_xl_cpds_128.safetensors')
return os.path.join(path_controlnet, 'fooocus_xl_cpds_128.safetensors')
def downloading_ip_adapters():
@@ -270,24 +269,24 @@ def downloading_ip_adapters():
load_file_from_url(
url='https://huggingface.co/lllyasviel/misc/resolve/main/clip_vision_vit_h.safetensors',
model_dir=clip_vision_models_path,
model_dir=path_clip_vision,
file_name='clip_vision_vit_h.safetensors'
)
results += [os.path.join(clip_vision_models_path, 'clip_vision_vit_h.safetensors')]
results += [os.path.join(path_clip_vision, 'clip_vision_vit_h.safetensors')]
load_file_from_url(
url='https://huggingface.co/lllyasviel/misc/resolve/main/fooocus_ip_negative.safetensors',
model_dir=controlnet_models_path,
model_dir=path_controlnet,
file_name='fooocus_ip_negative.safetensors'
)
results += [os.path.join(controlnet_models_path, 'fooocus_ip_negative.safetensors')]
results += [os.path.join(path_controlnet, 'fooocus_ip_negative.safetensors')]
load_file_from_url(
url='https://huggingface.co/lllyasviel/misc/resolve/main/ip-adapter-plus_sdxl_vit-h.bin',
model_dir=controlnet_models_path,
model_dir=path_controlnet,
file_name='ip-adapter-plus_sdxl_vit-h.bin'
)
results += [os.path.join(controlnet_models_path, 'ip-adapter-plus_sdxl_vit-h.bin')]
results += [os.path.join(path_controlnet, 'ip-adapter-plus_sdxl_vit-h.bin')]
return results
@@ -295,10 +294,10 @@ def downloading_ip_adapters():
def downloading_upscale_model():
load_file_from_url(
url='https://huggingface.co/lllyasviel/misc/resolve/main/fooocus_upscaler_s409985e5.bin',
model_dir=upscale_models_path,
model_dir=path_upscale_models,
file_name='fooocus_upscaler_s409985e5.bin'
)
return os.path.join(upscale_models_path, 'fooocus_upscaler_s409985e5.bin')
return os.path.join(path_upscale_models, 'fooocus_upscaler_s409985e5.bin')
update_all_model_names()
modules/core.py
+76 -8
View File
@@ -22,9 +22,10 @@ from nodes import VAEDecode, EmptyLatentImage, VAEEncode, VAEEncodeTiled, VAEDec
ControlNetApplyAdvanced
from fcbh_extras.nodes_freelunch import FreeU_V2
from fcbh.sample import prepare_mask
from modules.patch import patched_sampler_cfg_function, patched_model_function_wrapper
from modules.patch import patched_sampler_cfg_function
from fcbh.lora import model_lora_keys_unet, model_lora_keys_clip, load_lora
from modules.path import embeddings_path
from modules.config import path_embeddings
from modules.lora import load_dangerous_lora
opEmptyLatentImage = EmptyLatentImage()
@@ -37,11 +38,79 @@ opFreeU = FreeU_V2()
class StableDiffusionModel:
def __init__(self, unet, vae, clip, clip_vision):
def __init__(self, unet=None, vae=None, clip=None, clip_vision=None, filename=None):
self.unet = unet
self.vae = vae
self.clip = clip
self.clip_vision = clip_vision
self.filename = filename
self.unet_with_lora = unet
self.clip_with_lora = clip
self.visited_loras = ''
self.lora_key_map = {}
if self.unet is not None and self.clip is not None:
self.lora_key_map = model_lora_keys_unet(self.unet.model, self.lora_key_map)
self.lora_key_map = model_lora_keys_clip(self.clip.cond_stage_model, self.lora_key_map)
self.lora_key_map.update({x: x for x in self.unet.model.state_dict().keys()})
self.lora_key_map.update({x: x for x in self.clip.cond_stage_model.state_dict().keys()})
@torch.no_grad()
@torch.inference_mode()
def refresh_loras(self, loras):
assert isinstance(loras, list)
print(f'Request to load LoRAs {str(loras)} for model [{self.filename}].')
if self.visited_loras == str(loras):
return
self.visited_loras = str(loras)
loras_to_load = []
if self.unet is None:
return
for name, weight in loras:
if name == 'None':
continue
if os.path.exists(name):
lora_filename = name
else:
lora_filename = os.path.join(modules.config.path_loras, name)
if not os.path.exists(lora_filename):
print(f'Lora file not found: {lora_filename}')
continue
loras_to_load.append((lora_filename, weight))
self.unet_with_lora = self.unet.clone() if self.unet is not None else None
self.clip_with_lora = self.clip.clone() if self.clip is not None else None
for lora_filename, weight in loras_to_load:
lora = fcbh.utils.load_torch_file(lora_filename, safe_load=False)
lora_items = load_dangerous_lora(lora, self.lora_key_map)
if len(lora_items) == 0:
continue
print(f'Loaded LoRA [{lora_filename}] for model [{self.filename}] with {len(lora_items)} keys at weight {weight}.')
if self.unet_with_lora is not None:
loaded_unet_keys = self.unet_with_lora.add_patches(lora_items, weight)
else:
loaded_unet_keys = []
if self.clip_with_lora is not None:
loaded_clip_keys = self.clip_with_lora.add_patches(lora_items, weight)
else:
loaded_clip_keys = []
for item in lora_items:
if item not in set(list(loaded_unet_keys) + list(loaded_clip_keys)):
print("LoRA key skipped: ", item)
@torch.no_grad()
@@ -66,10 +135,9 @@ def apply_controlnet(positive, negative, control_net, image, strength, start_per
@torch.no_grad()
@torch.inference_mode()
def load_model(ckpt_filename):
unet, clip, vae, clip_vision = load_checkpoint_guess_config(ckpt_filename, embedding_directory=embeddings_path)
unet, clip, vae, clip_vision = load_checkpoint_guess_config(ckpt_filename, embedding_directory=path_embeddings)
unet.model_options['sampler_cfg_function'] = patched_sampler_cfg_function
unet.model_options['model_function_wrapper'] = patched_model_function_wrapper
return StableDiffusionModel(unet=unet, clip=clip, vae=vae, clip_vision=clip_vision)
return StableDiffusionModel(unet=unet, clip=clip, vae=vae, clip_vision=clip_vision, filename=ckpt_filename)
@torch.no_grad()
@@ -177,9 +245,9 @@ VAE_approx_models = {}
def get_previewer(model):
global VAE_approx_models
from modules.path import vae_approx_path
from modules.config import path_vae_approx
is_sdxl = isinstance(model.model.latent_format, fcbh.latent_formats.SDXL)
vae_approx_filename = os.path.join(vae_approx_path, 'xlvaeapp.pth' if is_sdxl else 'vaeapp_sd15.pth')
vae_approx_filename = os.path.join(path_vae_approx, 'xlvaeapp.pth' if is_sdxl else 'vaeapp_sd15.pth')
if vae_approx_filename in VAE_approx_models:
VAE_approx_model = VAE_approx_models[vae_approx_filename]
modules/default_pipeline.py
+46 -91
View File
@@ -2,7 +2,7 @@ import modules.core as core
import os
import torch
import modules.patch
import modules.path
import modules.config
import fcbh.model_management
import fcbh.latent_formats
import modules.inpaint_worker
@@ -13,14 +13,8 @@ from modules.expansion import FooocusExpansion
from modules.sample_hijack import clip_separate
xl_base: core.StableDiffusionModel = None
xl_base_hash = ''
xl_base_patched: core.StableDiffusionModel = None
xl_base_patched_hash = ''
xl_refiner: core.StableDiffusionModel = None
xl_refiner_hash = ''
model_base = core.StableDiffusionModel()
model_refiner = core.StableDiffusionModel()
final_expansion = None
final_unet = None
@@ -52,24 +46,9 @@ def refresh_controlnets(model_paths):
def assert_model_integrity():
error_message = None
if xl_base is None:
error_message = 'You have not selected SDXL base model.'
if xl_base_patched is None:
error_message = 'You have not selected SDXL base model.'
if not isinstance(xl_base.unet.model, SDXL):
if not isinstance(model_base.unet_with_lora.model, SDXL):
error_message = 'You have selected base model other than SDXL. This is not supported yet.'
if not isinstance(xl_base_patched.unet.model, SDXL):
error_message = 'You have selected base model other than SDXL. This is not supported yet.'
if xl_refiner is not None:
if xl_refiner.unet is None or xl_refiner.unet.model is None:
error_message = 'You have selected an invalid refiner!'
# elif not isinstance(xl_refiner.unet.model, SDXL) and not isinstance(xl_refiner.unet.model, SDXLRefiner):
# error_message = 'SD1.5 or 2.1 as refiner is not supported!'
if error_message is not None:
raise NotImplementedError(error_message)
@@ -79,82 +58,60 @@ def assert_model_integrity():
@torch.no_grad()
@torch.inference_mode()
def refresh_base_model(name):
global xl_base, xl_base_hash, xl_base_patched, xl_base_patched_hash
global model_base
filename = os.path.abspath(os.path.realpath(os.path.join(modules.path.modelfile_path, name)))
model_hash = filename
filename = os.path.abspath(os.path.realpath(os.path.join(modules.config.path_checkpoints, name)))
if xl_base_hash == model_hash:
if model_base.filename == filename:
return
xl_base = None
xl_base_hash = ''
xl_base_patched = None
xl_base_patched_hash = ''
xl_base = core.load_model(filename)
xl_base_hash = model_hash
print(f'Base model loaded: {model_hash}')
model_base = core.StableDiffusionModel()
model_base = core.load_model(filename)
print(f'Base model loaded: {model_base.filename}')
return
@torch.no_grad()
@torch.inference_mode()
def refresh_refiner_model(name):
global xl_refiner, xl_refiner_hash
global model_refiner
filename = os.path.abspath(os.path.realpath(os.path.join(modules.path.modelfile_path, name)))
model_hash = filename
filename = os.path.abspath(os.path.realpath(os.path.join(modules.config.path_checkpoints, name)))
if xl_refiner_hash == model_hash:
if model_refiner.filename == filename:
return
xl_refiner = None
xl_refiner_hash = ''
model_refiner = core.StableDiffusionModel()
if name == 'None':
print(f'Refiner unloaded.')
return
xl_refiner = core.load_model(filename)
xl_refiner_hash = model_hash
print(f'Refiner model loaded: {model_hash}')
model_refiner = core.load_model(filename)
print(f'Refiner model loaded: {model_refiner.filename}')
if isinstance(xl_refiner.unet.model, SDXL):
xl_refiner.clip = None
xl_refiner.vae = None
elif isinstance(xl_refiner.unet.model, SDXLRefiner):
xl_refiner.clip = None
xl_refiner.vae = None
if isinstance(model_refiner.unet.model, SDXL):
model_refiner.clip = None
model_refiner.vae = None
elif isinstance(model_refiner.unet.model, SDXLRefiner):
model_refiner.clip = None
model_refiner.vae = None
else:
xl_refiner.clip = None
model_refiner.clip = None
return
@torch.no_grad()
@torch.inference_mode()
def refresh_loras(loras):
global xl_base, xl_base_patched, xl_base_patched_hash
if xl_base_patched_hash == str(loras):
return
def refresh_loras(loras, base_model_additional_loras=None):
global model_base, model_refiner
model = xl_base
for name, weight in loras:
if name == 'None':
continue
if not isinstance(base_model_additional_loras, list):
base_model_additional_loras = []
if os.path.exists(name):
filename = name
else:
filename = os.path.join(modules.path.lorafile_path, name)
assert os.path.exists(filename), 'Lora file not found!'
model = core.load_sd_lora(model, filename, strength_model=weight, strength_clip=weight)
xl_base_patched = model
xl_base_patched_hash = str(loras)
print(f'LoRAs loaded: {xl_base_patched_hash}')
model_base.refresh_loras(loras + base_model_additional_loras)
model_refiner.refresh_loras(loras)
return
@@ -202,8 +159,7 @@ def clip_encode(texts, pool_top_k=1):
@torch.no_grad()
@torch.inference_mode()
def clear_all_caches():
xl_base.clip.fcs_cond_cache = {}
xl_base_patched.clip.fcs_cond_cache = {}
final_clip.fcs_cond_cache = {}
@torch.no_grad()
@@ -219,7 +175,7 @@ def prepare_text_encoder(async_call=True):
@torch.no_grad()
@torch.inference_mode()
def refresh_everything(refiner_model_name, base_model_name, loras):
def refresh_everything(refiner_model_name, base_model_name, loras, base_model_additional_loras=None):
global final_unet, final_clip, final_vae, final_refiner_unet, final_refiner_vae, final_expansion
final_unet = None
@@ -230,21 +186,20 @@ def refresh_everything(refiner_model_name, base_model_name, loras):
refresh_refiner_model(refiner_model_name)
refresh_base_model(base_model_name)
refresh_loras(loras)
refresh_loras(loras, base_model_additional_loras=base_model_additional_loras)
assert_model_integrity()
final_unet = xl_base_patched.unet
final_clip = xl_base_patched.clip
final_vae = xl_base_patched.vae
final_unet = model_base.unet_with_lora
final_clip = model_base.clip_with_lora
final_vae = model_base.vae
final_unet.model.diffusion_model.in_inpaint = False
if xl_refiner is not None:
final_refiner_unet = xl_refiner.unet
final_refiner_vae = xl_refiner.vae
final_refiner_unet = model_refiner.unet_with_lora
final_refiner_vae = model_refiner.vae
if final_refiner_unet is not None:
final_refiner_unet.model.diffusion_model.in_inpaint = False
if final_refiner_unet is not None:
final_refiner_unet.model.diffusion_model.in_inpaint = False
if final_expansion is None:
final_expansion = FooocusExpansion()
@@ -255,14 +210,14 @@ def refresh_everything(refiner_model_name, base_model_name, loras):
refresh_everything(
refiner_model_name=modules.path.default_refiner_model_name,
base_model_name=modules.path.default_base_model_name,
refiner_model_name=modules.config.default_refiner_model_name,
base_model_name=modules.config.default_base_model_name,
loras=[
(modules.path.default_lora_name, modules.path.default_lora_weight),
('None', modules.path.default_lora_weight),
('None', modules.path.default_lora_weight),
('None', modules.path.default_lora_weight),
('None', modules.path.default_lora_weight)
(modules.config.default_lora_name, modules.config.default_lora_weight),
('None', modules.config.default_lora_weight),
('None', modules.config.default_lora_weight),
('None', modules.config.default_lora_weight),
('None', modules.config.default_lora_weight)
]
)
modules/expansion.py
+4 -4
View File
@@ -12,7 +12,7 @@ import fcbh.model_management as model_management
from transformers.generation.logits_process import LogitsProcessorList
from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed
from modules.path import fooocus_expansion_path
from modules.config import path_fooocus_expansion
from fcbh.model_patcher import ModelPatcher
@@ -36,9 +36,9 @@ def remove_pattern(x, pattern):
class FooocusExpansion:
def __init__(self):
self.tokenizer = AutoTokenizer.from_pretrained(fooocus_expansion_path)
self.tokenizer = AutoTokenizer.from_pretrained(path_fooocus_expansion)
positive_words = open(os.path.join(fooocus_expansion_path, 'positive.txt'),
positive_words = open(os.path.join(path_fooocus_expansion, 'positive.txt'),
encoding='utf-8').read().splitlines()
positive_words = ['Ġ' + x.lower() for x in positive_words if x != '']
@@ -59,7 +59,7 @@ class FooocusExpansion:
# t198 = self.tokenizer('\n', return_tensors="np")
# eos = self.tokenizer.eos_token_id
self.model = AutoModelForCausalLM.from_pretrained(fooocus_expansion_path)
self.model = AutoModelForCausalLM.from_pretrained(path_fooocus_expansion)
self.model.eval()
load_device = model_management.text_encoder_device()
modules/flags.py
+1 -1
View File
@@ -12,7 +12,7 @@ uov_list = [
KSAMPLER_NAMES = ["euler", "euler_ancestral", "heun", "dpm_2", "dpm_2_ancestral",
"lms", "dpm_fast", "dpm_adaptive", "dpmpp_2s_ancestral", "dpmpp_sde", "dpmpp_sde_gpu",
"dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm"]
"dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm", "lcm"]
SCHEDULER_NAMES = ["normal", "karras", "exponential", "sgm_uniform", "simple", "ddim_uniform"]
SAMPLER_NAMES = KSAMPLER_NAMES + ["ddim", "uni_pc", "uni_pc_bh2"]
modules/inpaint_worker.py
+1 -1
View File
@@ -187,7 +187,7 @@ class InpaintWorker:
feed = torch.cat([
latent_mask,
pipeline.xl_base_patched.unet.model.process_latent_in(latent_inpaint)
pipeline.final_unet.model.process_latent_in(latent_inpaint)
], dim=1)
inpaint_head.to(device=feed.device, dtype=feed.dtype)
modules/lora.py
+142
View File
@@ -0,0 +1,142 @@
def load_dangerous_lora(lora, to_load):
patch_dict = {}
loaded_keys = set()
for x in to_load:
real_load_key = to_load[x]
if real_load_key in lora:
patch_dict[real_load_key] = lora[real_load_key]
loaded_keys.add(real_load_key)
continue
alpha_name = "{}.alpha".format(x)
alpha = None
if alpha_name in lora.keys():
alpha = lora[alpha_name].item()
loaded_keys.add(alpha_name)
regular_lora = "{}.lora_up.weight".format(x)
diffusers_lora = "{}_lora.up.weight".format(x)
transformers_lora = "{}.lora_linear_layer.up.weight".format(x)
A_name = None
if regular_lora in lora.keys():
A_name = regular_lora
B_name = "{}.lora_down.weight".format(x)
mid_name = "{}.lora_mid.weight".format(x)
elif diffusers_lora in lora.keys():
A_name = diffusers_lora
B_name = "{}_lora.down.weight".format(x)
mid_name = None
elif transformers_lora in lora.keys():
A_name = transformers_lora
B_name ="{}.lora_linear_layer.down.weight".format(x)
mid_name = None
if A_name is not None:
mid = None
if mid_name is not None and mid_name in lora.keys():
mid = lora[mid_name]
loaded_keys.add(mid_name)
patch_dict[to_load[x]] = (lora[A_name], lora[B_name], alpha, mid)
loaded_keys.add(A_name)
loaded_keys.add(B_name)
######## loha
hada_w1_a_name = "{}.hada_w1_a".format(x)
hada_w1_b_name = "{}.hada_w1_b".format(x)
hada_w2_a_name = "{}.hada_w2_a".format(x)
hada_w2_b_name = "{}.hada_w2_b".format(x)
hada_t1_name = "{}.hada_t1".format(x)
hada_t2_name = "{}.hada_t2".format(x)
if hada_w1_a_name in lora.keys():
hada_t1 = None
hada_t2 = None
if hada_t1_name in lora.keys():
hada_t1 = lora[hada_t1_name]
hada_t2 = lora[hada_t2_name]
loaded_keys.add(hada_t1_name)
loaded_keys.add(hada_t2_name)
patch_dict[to_load[x]] = (lora[hada_w1_a_name], lora[hada_w1_b_name], alpha, lora[hada_w2_a_name], lora[hada_w2_b_name], hada_t1, hada_t2)
loaded_keys.add(hada_w1_a_name)
loaded_keys.add(hada_w1_b_name)
loaded_keys.add(hada_w2_a_name)
loaded_keys.add(hada_w2_b_name)
######## lokr
lokr_w1_name = "{}.lokr_w1".format(x)
lokr_w2_name = "{}.lokr_w2".format(x)
lokr_w1_a_name = "{}.lokr_w1_a".format(x)
lokr_w1_b_name = "{}.lokr_w1_b".format(x)
lokr_t2_name = "{}.lokr_t2".format(x)
lokr_w2_a_name = "{}.lokr_w2_a".format(x)
lokr_w2_b_name = "{}.lokr_w2_b".format(x)
lokr_w1 = None
if lokr_w1_name in lora.keys():
lokr_w1 = lora[lokr_w1_name]
loaded_keys.add(lokr_w1_name)
lokr_w2 = None
if lokr_w2_name in lora.keys():
lokr_w2 = lora[lokr_w2_name]
loaded_keys.add(lokr_w2_name)
lokr_w1_a = None
if lokr_w1_a_name in lora.keys():
lokr_w1_a = lora[lokr_w1_a_name]
loaded_keys.add(lokr_w1_a_name)
lokr_w1_b = None
if lokr_w1_b_name in lora.keys():
lokr_w1_b = lora[lokr_w1_b_name]
loaded_keys.add(lokr_w1_b_name)
lokr_w2_a = None
if lokr_w2_a_name in lora.keys():
lokr_w2_a = lora[lokr_w2_a_name]
loaded_keys.add(lokr_w2_a_name)
lokr_w2_b = None
if lokr_w2_b_name in lora.keys():
lokr_w2_b = lora[lokr_w2_b_name]
loaded_keys.add(lokr_w2_b_name)
lokr_t2 = None
if lokr_t2_name in lora.keys():
lokr_t2 = lora[lokr_t2_name]
loaded_keys.add(lokr_t2_name)
if (lokr_w1 is not None) or (lokr_w2 is not None) or (lokr_w1_a is not None) or (lokr_w2_a is not None):
patch_dict[to_load[x]] = (lokr_w1, lokr_w2, alpha, lokr_w1_a, lokr_w1_b, lokr_w2_a, lokr_w2_b, lokr_t2)
w_norm_name = "{}.w_norm".format(x)
b_norm_name = "{}.b_norm".format(x)
w_norm = lora.get(w_norm_name, None)
b_norm = lora.get(b_norm_name, None)
if w_norm is not None:
loaded_keys.add(w_norm_name)
patch_dict[to_load[x]] = (w_norm,)
if b_norm is not None:
loaded_keys.add(b_norm_name)
patch_dict["{}.bias".format(to_load[x][:-len(".weight")])] = (b_norm,)
diff_name = "{}.diff".format(x)
diff_weight = lora.get(diff_name, None)
if diff_weight is not None:
patch_dict[to_load[x]] = (diff_weight,)
loaded_keys.add(diff_name)
diff_bias_name = "{}.diff_b".format(x)
diff_bias = lora.get(diff_bias_name, None)
if diff_bias is not None:
patch_dict["{}.bias".format(to_load[x][:-len(".weight")])] = (diff_bias,)
loaded_keys.add(diff_bias_name)
for x in lora.keys():
if x not in loaded_keys:
return {}
return patch_dict
modules/patch.py
+74 -87
View File
@@ -1,11 +1,9 @@
import contextlib
import os
import torch
import time
import fcbh.model_base
import fcbh.ldm.modules.diffusionmodules.openaimodel
import fcbh.samplers
import fcbh.k_diffusion.external
import fcbh.model_management
import modules.anisotropic as anisotropic
import fcbh.ldm.modules.attention
@@ -19,15 +17,13 @@ import fcbh.cldm.cldm
import fcbh.model_patcher
import fcbh.samplers
import fcbh.cli_args
import args_manager
import modules.advanced_parameters as advanced_parameters
import warnings
import safetensors.torch
import modules.constants as constants
from fcbh.k_diffusion import utils
from fcbh.k_diffusion.sampling import BatchedBrownianTree
from fcbh.ldm.modules.diffusionmodules.openaimodel import timestep_embedding, forward_timestep_embed
from fcbh.ldm.modules.diffusionmodules.openaimodel import forward_timestep_embed, apply_control, timestep_embedding
sharpness = 2.0
@@ -36,10 +32,7 @@ adm_scaler_end = 0.3
positive_adm_scale = 1.5
negative_adm_scale = 0.8
cfg_x0 = 0.0
cfg_s = 1.0
cfg_cin = 1.0
adaptive_cfg = 0.7
adaptive_cfg = 7.0
eps_record = None
@@ -161,6 +154,34 @@ def calculate_weight_patched(self, patches, weight, key):
return weight
class BrownianTreeNoiseSamplerPatched:
transform = None
tree = None
global_sigma_min = 1.0
global_sigma_max = 1.0
@staticmethod
def global_init(x, sigma_min, sigma_max, seed=None, transform=lambda x: x, cpu=False):
t0, t1 = transform(torch.as_tensor(sigma_min)), transform(torch.as_tensor(sigma_max))
BrownianTreeNoiseSamplerPatched.transform = transform
BrownianTreeNoiseSamplerPatched.tree = BatchedBrownianTree(x, t0, t1, seed, cpu=cpu)
BrownianTreeNoiseSamplerPatched.global_sigma_min = sigma_min
BrownianTreeNoiseSamplerPatched.global_sigma_max = sigma_max
def __init__(self, *args, **kwargs):
pass
@staticmethod
def __call__(sigma, sigma_next):
transform = BrownianTreeNoiseSamplerPatched.transform
tree = BrownianTreeNoiseSamplerPatched.tree
t0, t1 = transform(torch.as_tensor(sigma)), transform(torch.as_tensor(sigma_next))
return tree(t0, t1) / (t1 - t0).abs().sqrt()
def compute_cfg(uncond, cond, cfg_scale, t):
global adaptive_cfg
@@ -169,46 +190,36 @@ def compute_cfg(uncond, cond, cfg_scale, t):
real_eps = uncond + real_cfg * (cond - uncond)
if cfg_scale < adaptive_cfg:
if cfg_scale > adaptive_cfg:
mimicked_eps = uncond + mimic_cfg * (cond - uncond)
return real_eps * t + mimicked_eps * (1 - t)
else:
return real_eps
mimicked_eps = uncond + mimic_cfg * (cond - uncond)
return real_eps * t + mimicked_eps * (1 - t)
def patched_sampler_cfg_function(args):
global cfg_x0, cfg_s
global eps_record
positive_eps = args['cond']
negative_eps = args['uncond']
cfg_scale = args['cond_scale']
positive_x0 = args['input'] - positive_eps
positive_x0 = args['cond'] * cfg_s + cfg_x0
t = 1.0 - (args['timestep'] / 999.0)[:, None, None, None].clone()
sigma = args['sigma']
t = 1.0 - (sigma / BrownianTreeNoiseSamplerPatched.global_sigma_max)[:, None, None, None]
t = t.clip(0, 1).to(sigma)
alpha = 0.001 * sharpness * t
positive_eps_degraded = anisotropic.adaptive_anisotropic_filter(x=positive_eps, g=positive_x0)
positive_eps_degraded_weighted = positive_eps_degraded * alpha + positive_eps * (1.0 - alpha)
return compute_cfg(uncond=negative_eps, cond=positive_eps_degraded_weighted, cfg_scale=cfg_scale, t=t)
final_eps = compute_cfg(uncond=negative_eps, cond=positive_eps_degraded_weighted, cfg_scale=cfg_scale, t=t)
def patched_discrete_eps_ddpm_denoiser_forward(self, input, sigma, **kwargs):
global cfg_x0, cfg_s, cfg_cin, eps_record
c_out, c_in = [utils.append_dims(x, input.ndim) for x in self.get_scalings(sigma)]
cfg_x0, cfg_s, cfg_cin = input, c_out, c_in
eps = self.get_eps(input * c_in, self.sigma_to_t(sigma), **kwargs)
if eps_record is not None:
eps_record = eps.clone().cpu()
return input + eps * c_out
eps_record = (final_eps / sigma).cpu()
def patched_model_function_wrapper(func, args):
x = args['input']
t = args['timestep']
c = args['c']
return func(x, t, **c)
return final_eps
def sdxl_encode_adm_patched(self, **kwargs):
@@ -249,36 +260,44 @@ def sdxl_encode_adm_patched(self, **kwargs):
def encode_token_weights_patched_with_a1111_method(self, token_weight_pairs):
to_encode = list(self.empty_tokens)
to_encode = list()
max_token_len = 0
has_weights = False
for x in token_weight_pairs:
tokens = list(map(lambda a: a[0], x))
max_token_len = max(len(tokens), max_token_len)
has_weights = has_weights or not all(map(lambda a: a[1] == 1.0, x))
to_encode.append(tokens)
out, pooled = self.encode(to_encode)
sections = len(to_encode)
if has_weights or sections == 0:
to_encode.append(fcbh.sd1_clip.gen_empty_tokens(self.special_tokens, max_token_len))
z_empty = out[0:1]
if pooled.shape[0] > 1:
first_pooled = pooled[1:2]
out, pooled = self.encode(to_encode)
if pooled is not None:
first_pooled = pooled[0:1].cpu()
else:
first_pooled = pooled[0:1]
first_pooled = pooled
output = []
for k in range(1, out.shape[0]):
for k in range(0, sections):
z = out[k:k + 1]
original_mean = z.mean()
for i in range(len(z)):
for j in range(len(z[i])):
weight = token_weight_pairs[k - 1][j][1]
z[i][j] = (z[i][j] - z_empty[0][j]) * weight + z_empty[0][j]
new_mean = z.mean()
z = z * (original_mean / new_mean)
if has_weights:
original_mean = z.mean()
z_empty = out[-1]
for i in range(len(z)):
for j in range(len(z[i])):
weight = token_weight_pairs[k][j][1]
if weight != 1.0:
z[i][j] = (z[i][j] - z_empty[j]) * weight + z_empty[j]
new_mean = z.mean()
z = z * (original_mean / new_mean)
output.append(z)
if len(output) == 0:
return z_empty.cpu(), first_pooled.cpu()
return torch.cat(output, dim=-2).cpu(), first_pooled.cpu()
return out[-1:].cpu(), first_pooled
return torch.cat(output, dim=-2).cpu(), first_pooled
def patched_KSamplerX0Inpaint_forward(self, x, sigma, uncond, cond, cond_scale, denoise_mask, model_options={}, seed=None):
@@ -287,7 +306,7 @@ def patched_KSamplerX0Inpaint_forward(self, x, sigma, uncond, cond, cond_scale,
# avoid bad results by using different seeds.
self.energy_generator = torch.Generator(device='cpu').manual_seed((seed + 1) % constants.MAX_SEED)
latent_processor = self.inner_model.inner_model.inner_model.process_latent_in
latent_processor = self.inner_model.inner_model.process_latent_in
inpaint_latent = latent_processor(inpaint_worker.current_task.latent).to(x)
inpaint_mask = inpaint_worker.current_task.latent_mask.to(x)
energy_sigma = sigma.reshape([sigma.shape[0]] + [1] * (len(x.shape) - 1))
@@ -312,29 +331,6 @@ def patched_KSamplerX0Inpaint_forward(self, x, sigma, uncond, cond, cond_scale,
return out
class BrownianTreeNoiseSamplerPatched:
transform = None
tree = None
@staticmethod
def global_init(x, sigma_min, sigma_max, seed=None, transform=lambda x: x, cpu=False):
t0, t1 = transform(torch.as_tensor(sigma_min)), transform(torch.as_tensor(sigma_max))
BrownianTreeNoiseSamplerPatched.transform = transform
BrownianTreeNoiseSamplerPatched.tree = BatchedBrownianTree(x, t0, t1, seed, cpu=cpu)
def __init__(self, *args, **kwargs):
pass
@staticmethod
def __call__(sigma, sigma_next):
transform = BrownianTreeNoiseSamplerPatched.transform
tree = BrownianTreeNoiseSamplerPatched.tree
t0, t1 = transform(torch.as_tensor(sigma)), transform(torch.as_tensor(sigma_next))
return tree(t0, t1) / (t1 - t0).abs().sqrt()
def timed_adm(y, timesteps):
if isinstance(y, torch.Tensor) and int(y.dim()) == 2 and int(y.shape[1]) == 5632:
y_mask = (timesteps > 999.0 * (1.0 - float(adm_scaler_end))).to(y)[..., None]
@@ -411,25 +407,17 @@ def patched_unet_forward(self, x, timesteps=None, context=None, y=None, control=
h = h + inpaint_fix.to(h)
inpaint_fix = None
if control is not None and 'input' in control and len(control['input']) > 0:
ctrl = control['input'].pop()
if ctrl is not None:
h += ctrl
h = apply_control(h, control, 'input')
hs.append(h)
transformer_options["block"] = ("middle", 0)
h = forward_timestep_embed(self.middle_block, h, emb, context, transformer_options)
if control is not None and 'middle' in control and len(control['middle']) > 0:
ctrl = control['middle'].pop()
if ctrl is not None:
h += ctrl
h = apply_control(h, control, 'middle')
for id, module in enumerate(self.output_blocks):
transformer_options["block"] = ("output", id)
hsp = hs.pop()
if control is not None and 'output' in control and len(control['output']) > 0:
ctrl = control['output'].pop()
if ctrl is not None:
hsp += ctrl
hsp = apply_control(hsp, control, 'output')
if "output_block_patch" in transformer_patches:
patch = transformer_patches["output_block_patch"]
@@ -501,7 +489,6 @@ def patch_all():
fcbh.model_patcher.ModelPatcher.calculate_weight = calculate_weight_patched
fcbh.cldm.cldm.ControlNet.forward = patched_cldm_forward
fcbh.ldm.modules.diffusionmodules.openaimodel.UNetModel.forward = patched_unet_forward
fcbh.k_diffusion.external.DiscreteEpsDDPMDenoiser.forward = patched_discrete_eps_ddpm_denoiser_forward
fcbh.model_base.SDXL.encode_adm = sdxl_encode_adm_patched
fcbh.sd1_clip.ClipTokenWeightEncoder.encode_token_weights = encode_token_weights_patched_with_a1111_method
fcbh.samplers.KSamplerX0Inpaint.forward = patched_KSamplerX0Inpaint_forward
modules/private_logger.py
+3 -3
View File
@@ -1,19 +1,19 @@
import os
import modules.path
import modules.config
from PIL import Image
from modules.util import generate_temp_filename
def get_current_html_path():
date_string, local_temp_filename, only_name = generate_temp_filename(folder=modules.path.temp_outputs_path,
date_string, local_temp_filename, only_name = generate_temp_filename(folder=modules.config.path_outputs,
extension='png')
html_name = os.path.join(os.path.dirname(local_temp_filename), 'log.html')
return html_name
def log(img, dic, single_line_number=3):
date_string, local_temp_filename, only_name = generate_temp_filename(folder=modules.path.temp_outputs_path, extension='png')
date_string, local_temp_filename, only_name = generate_temp_filename(folder=modules.config.path_outputs, extension='png')
os.makedirs(os.path.dirname(local_temp_filename), exist_ok=True)
Image.fromarray(img).save(local_temp_filename)
html_name = os.path.join(os.path.dirname(local_temp_filename), 'log.html')
modules/sample_hijack.py
+5 -5
View File
@@ -92,8 +92,8 @@ def sample_hacked(model, noise, positive, negative, cfg, device, sampler, sigmas
model_wrap = wrap_model(model)
calculate_start_end_timesteps(model_wrap, negative)
calculate_start_end_timesteps(model_wrap, positive)
calculate_start_end_timesteps(model, negative)
calculate_start_end_timesteps(model, positive)
#make sure each cond area has an opposite one with the same area
for c in positive:
@@ -101,8 +101,8 @@ def sample_hacked(model, noise, positive, negative, cfg, device, sampler, sigmas
for c in negative:
create_cond_with_same_area_if_none(positive, c)
# pre_run_control(model_wrap, negative + positive)
pre_run_control(model_wrap, positive) # negative is not necessary in Fooocus, 0.5s faster.
# pre_run_control(model, negative + positive)
pre_run_control(model, positive) # negative is not necessary in Fooocus, 0.5s faster.
apply_empty_x_to_equal_area(list(filter(lambda c: c.get('control_apply_to_uncond', False) == True, positive)), negative, 'control', lambda cond_cnets, x: cond_cnets[x])
apply_empty_x_to_equal_area(positive, negative, 'gligen', lambda cond_cnets, x: cond_cnets[x])
@@ -136,7 +136,7 @@ def sample_hacked(model, noise, positive, negative, cfg, device, sampler, sigmas
fcbh.model_management.load_models_gpu([current_refiner] + models, fcbh.model_management.batch_area_memory(
noise.shape[0] * noise.shape[2] * noise.shape[3]) + inference_memory)
model_wrap.inner_model.inner_model = current_refiner.model
model_wrap.inner_model = current_refiner.model
print('Refiner Swapped')
return
modules/sdxl_styles.py
+1 -1
View File
@@ -5,7 +5,7 @@ import json
from modules.util import get_files_from_folder
# cannot use modules.path - validators causing circular imports
# cannot use modules.config - validators causing circular imports
styles_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../sdxl_styles/'))
wildcards_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../wildcards/'))
wildcards_max_bfs_depth = 64
modules/upscaler.py
+2 -2
View File
@@ -4,9 +4,9 @@ import torch
from fcbh_extras.chainner_models.architecture.RRDB import RRDBNet as ESRGAN
from fcbh_extras.nodes_upscale_model import ImageUpscaleWithModel
from collections import OrderedDict
from modules.path import upscale_models_path
from modules.config import path_upscale_models
model_filename = os.path.join(upscale_models_path, 'fooocus_upscaler_s409985e5.bin')
model_filename = os.path.join(path_upscale_models, 'fooocus_upscaler_s409985e5.bin')
opImageUpscaleWithModel = ImageUpscaleWithModel()
model = None