Foreign-Mirrors/Fooocus
1
0
Fork 0
mirror of https://github.com/lllyasviel/Fooocus.git synced 2026-08-16 13:13:16 +02:00
Code Issues Packages Projects Releases Wiki Activity

2.1.790

Browse Source 
2.1.790
This commit is contained in:
lllyasviel
2023-11-11 22:13:13 -08:00
committed by GitHub
parent fd4a5b2eaf
commit 2bef62c545
10 changed files with 184 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fooocus_extras/face_crop.py
+50
View File
@@ -0,0 +1,50 @@
import cv2
import numpy as np
import modules.config
import facexlib.utils.face_restoration_helper as face_restoration_helper
faceRestoreHelper = None
def align_warp_face(self, landmark, border_mode='reflect'):
affine_matrix = cv2.estimateAffinePartial2D(landmark, self.face_template, method=cv2.LMEDS)[0]
self.affine_matrices.append(affine_matrix)
if border_mode == 'constant':
border_mode = cv2.BORDER_CONSTANT
elif border_mode == 'reflect101':
border_mode = cv2.BORDER_REFLECT101
elif border_mode == 'reflect':
border_mode = cv2.BORDER_REFLECT
input_img = self.input_img
cropped_face = cv2.warpAffine(input_img, affine_matrix, self.face_size,
borderMode=border_mode, borderValue=(135, 133, 132))
return cropped_face
def crop_image(img_rgb):
global faceRestoreHelper
if faceRestoreHelper is None:
faceRestoreHelper = face_restoration_helper.FaceRestoreHelper(
upscale_factor=1,
model_rootpath=modules.config.path_controlnet,
device='cpu' # use cpu is safer since we are out of fcbh management
)
faceRestoreHelper.clean_all()
faceRestoreHelper.read_image(np.ascontiguousarray(img_rgb[:, :, ::-1].copy()))
faceRestoreHelper.get_face_landmarks_5()
landmarks = faceRestoreHelper.all_landmarks_5
# landmarks are already sorted with confidence.
if len(landmarks) == 0:
print('No face detected')
return img_rgb
else:
print(f'Detected {len(landmarks)} faces')
result = align_warp_face(faceRestoreHelper, landmarks[0])
return np.ascontiguousarray(result[:, :, ::-1].copy())
fooocus_extras/ip_adapter.py
+73 -71
View File
@@ -84,26 +84,20 @@ class IPAdapterModel(torch.nn.Module):
clip_vision: fcbh.clip_vision.ClipVisionModel = None
ip_negative: torch.Tensor = None
image_proj_model: ModelPatcher = None
ip_layers: ModelPatcher = None
ip_adapter: IPAdapterModel = None
ip_unconds = None
ip_adapters: dict = {}
def load_ip_adapter(clip_vision_path, ip_negative_path, ip_adapter_path):
global clip_vision, image_proj_model, ip_layers, ip_negative, ip_adapter, ip_unconds
global clip_vision, ip_negative, ip_adapters
if clip_vision_path is None:
return
if ip_negative_path is None:
return
if ip_adapter_path is None:
return
if clip_vision is not None and image_proj_model is not None and ip_layers is not None and ip_negative is not None:
return
if clip_vision is None and isinstance(clip_vision_path, str):
clip_vision = fcbh.clip_vision.load(clip_vision_path)
ip_negative = sf.load_file(ip_negative_path)['data']
clip_vision = fcbh.clip_vision.load(clip_vision_path)
if ip_negative is None and isinstance(ip_negative_path, str):
ip_negative = sf.load_file(ip_negative_path)['data']
if not isinstance(ip_adapter_path, str) or ip_adapter_path in ip_adapters:
return
load_device = model_management.get_torch_device()
offload_device = torch.device('cpu')
@@ -141,7 +135,13 @@ def load_ip_adapter(clip_vision_path, ip_negative_path, ip_adapter_path):
ip_layers = ModelPatcher(model=ip_adapter.ip_layers, load_device=load_device,
offload_device=offload_device)
ip_unconds = None
ip_adapters[ip_adapter_path] = dict(
ip_adapter=ip_adapter,
image_proj_model=image_proj_model,
ip_layers=ip_layers,
ip_unconds=None
)
return
@@ -161,8 +161,9 @@ def clip_preprocess(image):
@torch.no_grad()
@torch.inference_mode()
def preprocess(img):
global ip_unconds
def preprocess(img, ip_adapter_path):
global ip_adapters
entry = ip_adapters[ip_adapter_path]
fcbh.model_management.load_model_gpu(clip_vision.patcher)
pixel_values = clip_preprocess(numpy_to_pytorch(img).to(clip_vision.load_device))
@@ -175,6 +176,11 @@ def preprocess(img):
with precision_scope(fcbh.model_management.get_autocast_device(clip_vision.load_device), torch.float32):
outputs = clip_vision.model(pixel_values=pixel_values, output_hidden_states=True)
ip_adapter = entry['ip_adapter']
ip_layers = entry['ip_layers']
image_proj_model = entry['image_proj_model']
ip_unconds = entry['ip_unconds']
if ip_adapter.plus:
cond = outputs.hidden_states[-2]
else:
@@ -190,9 +196,11 @@ def preprocess(img):
if ip_unconds is None:
uncond = ip_negative.to(device=ip_adapter.load_device, dtype=ip_adapter.dtype)
ip_unconds = [m(uncond).cpu() for m in ip_layers.model.to_kvs]
entry['ip_unconds'] = ip_unconds
ip_conds = [m(cond).cpu() for m in ip_layers.model.to_kvs]
return ip_conds
return ip_conds, ip_unconds
@torch.no_grad()
@@ -206,46 +214,46 @@ def patch_model(model, tasks):
current_step = float(model.model.diffusion_model.current_step.detach().cpu().numpy()[0])
cond_or_uncond = extra_options['cond_or_uncond']
with torch.autocast("cuda", dtype=ip_adapter.dtype):
q = n
k = [context_attn2]
v = [value_attn2]
b, _, _ = q.shape
q = n
k = [context_attn2]
v = [value_attn2]
b, _, _ = q.shape
for ip_conds, cn_stop, cn_weight in tasks:
if current_step < cn_stop:
ip_k_c = ip_conds[ip_index * 2].to(q)
ip_v_c = ip_conds[ip_index * 2 + 1].to(q)
ip_k_uc = ip_unconds[ip_index * 2].to(q)
ip_v_uc = ip_unconds[ip_index * 2 + 1].to(q)
for (cs, ucs), cn_stop, cn_weight in tasks:
if current_step < cn_stop:
ip_k_c = cs[ip_index * 2].to(q)
ip_v_c = cs[ip_index * 2 + 1].to(q)
ip_k_uc = ucs[ip_index * 2].to(q)
ip_v_uc = ucs[ip_index * 2 + 1].to(q)
ip_k = torch.cat([(ip_k_c, ip_k_uc)[i] for i in cond_or_uncond], dim=0)
ip_v = torch.cat([(ip_v_c, ip_v_uc)[i] for i in cond_or_uncond], dim=0)
ip_k = torch.cat([(ip_k_c, ip_k_uc)[i] for i in cond_or_uncond], dim=0)
ip_v = torch.cat([(ip_v_c, ip_v_uc)[i] for i in cond_or_uncond], dim=0)
# Midjourney's attention formulation of image prompt (non-official reimplementation)
# Written by Lvmin Zhang at Stanford University, 2023 Dec
# For non-commercial use only - if you use this in commercial project then
# probably it has some intellectual property issues.
# Contact lvminzhang@acm.org if you are not sure.
# Midjourney's attention formulation of image prompt (non-official reimplementation)
# Written by Lvmin Zhang at Stanford University, 2023 Dec
# For non-commercial use only - if you use this in commercial project then
# probably it has some intellectual property issues.
# Contact lvminzhang@acm.org if you are not sure.
# Below is the sensitive part with potential intellectual property issues.
# Below is the sensitive part with potential intellectual property issues.
ip_v_mean = torch.mean(ip_v, dim=1, keepdim=True)
ip_v_offset = ip_v - ip_v_mean
ip_v_mean = torch.mean(ip_v, dim=1, keepdim=True)
ip_v_offset = ip_v - ip_v_mean
B, F, C = ip_k.shape
channel_penalty = float(C) / 1280.0
weight = cn_weight * channel_penalty
B, F, C = ip_k.shape
channel_penalty = float(C) / 1280.0
weight = cn_weight * channel_penalty
ip_k = ip_k * weight
ip_v = ip_v_offset + ip_v_mean * weight
ip_k = ip_k * weight
ip_v = ip_v_offset + ip_v_mean * weight
k.append(ip_k)
v.append(ip_v)
k.append(ip_k)
v.append(ip_v)
k = torch.cat(k, dim=1)
v = torch.cat(v, dim=1)
out = sdp(q, k, v, extra_options)
k = torch.cat(k, dim=1)
v = torch.cat(v, dim=1)
out = sdp(q, k, v, extra_options)
return out.to(dtype=org_dtype)
return patcher
@@ -260,27 +268,21 @@ def patch_model(model, tasks):
to["patches_replace"]["attn2"][key] = make_attn_patcher(number)
number = 0
if not ip_adapter.sdxl:
for id in [1, 2, 4, 5, 7, 8]: # id of input_blocks that have cross attention
set_model_patch_replace(new_model, number, ("input", id))
number += 1
for id in [3, 4, 5, 6, 7, 8, 9, 10, 11]: # id of output_blocks that have cross attention
set_model_patch_replace(new_model, number, ("output", id))
number += 1
set_model_patch_replace(new_model, number, ("middle", 0))
else:
for id in [4, 5, 7, 8]: # id of input_blocks that have cross attention
block_indices = range(2) if id in [4, 5] else range(10) # transformer_depth
for index in block_indices:
set_model_patch_replace(new_model, number, ("input", id, index))
number += 1
for id in range(6): # id of output_blocks that have cross attention
block_indices = range(2) if id in [3, 4, 5] else range(10) # transformer_depth
for index in block_indices:
set_model_patch_replace(new_model, number, ("output", id, index))
number += 1
for index in range(10):
set_model_patch_replace(new_model, number, ("middle", 0, index))
for id in [4, 5, 7, 8]:
block_indices = range(2) if id in [4, 5] else range(10)
for index in block_indices:
set_model_patch_replace(new_model, number, ("input", id, index))
number += 1
for id in range(6):
block_indices = range(2) if id in [3, 4, 5] else range(10)
for index in block_indices:
set_model_patch_replace(new_model, number, ("output", id, index))
number += 1
for index in range(10):
set_model_patch_replace(new_model, number, ("middle", 0, index))
number += 1
return new_model