Merge pull request #3507 from lllyasviel/develop

Release 2.5.5

.github/CODEOWNERS

@@ -1 +1 @@
-*       @mashb1t
+*       @lllyasviel

extras/BLIP/models/blip.py

@@ -216,9 +216,9 @@ def is_url(url_or_filename):
 def load_checkpoint(model,url_or_filename):
     if is_url(url_or_filename):
         cached_file = download_cached_file(url_or_filename, check_hash=False, progress=True)
-        checkpoint = torch.load(cached_file, map_location='cpu') 
+        checkpoint = torch.load(cached_file, map_location='cpu', weights_only=True) 
     elif os.path.isfile(url_or_filename):        
-        checkpoint = torch.load(url_or_filename, map_location='cpu') 
+        checkpoint = torch.load(url_or_filename, map_location='cpu', weights_only=True) 
     else:
         raise RuntimeError('checkpoint url or path is invalid')
         

extras/BLIP/models/blip_nlvr.py

@@ -78,9 +78,9 @@ def blip_nlvr(pretrained='',**kwargs):
 def load_checkpoint(model,url_or_filename):
     if is_url(url_or_filename):
         cached_file = download_cached_file(url_or_filename, check_hash=False, progress=True)
-        checkpoint = torch.load(cached_file, map_location='cpu') 
+        checkpoint = torch.load(cached_file, map_location='cpu', weights_only=True) 
     elif os.path.isfile(url_or_filename):        
-        checkpoint = torch.load(url_or_filename, map_location='cpu') 
+        checkpoint = torch.load(url_or_filename, map_location='cpu', weights_only=True) 
     else:
         raise RuntimeError('checkpoint url or path is invalid')
     state_dict = checkpoint['model']

extras/facexlib/detection/__init__.py

@@ -19,7 +19,7 @@ def init_detection_model(model_name, half=False, device='cuda', model_rootpath=N
         url=model_url, model_dir='facexlib/weights', progress=True, file_name=None, save_dir=model_rootpath)
 
     # TODO: clean pretrained model
-    load_net = torch.load(model_path, map_location=lambda storage, loc: storage)
+    load_net = torch.load(model_path, map_location=lambda storage, loc: storage, weights_only=True)
     # remove unnecessary 'module.'
     for k, v in deepcopy(load_net).items():
         if k.startswith('module.'):

extras/facexlib/parsing/__init__.py

@@ -17,7 +17,7 @@ def init_parsing_model(model_name='bisenet', half=False, device='cuda', model_ro
 
     model_path = load_file_from_url(
         url=model_url, model_dir='facexlib/weights', progress=True, file_name=None, save_dir=model_rootpath)
-    load_net = torch.load(model_path, map_location=lambda storage, loc: storage)
+    load_net = torch.load(model_path, map_location=lambda storage, loc: storage, weights_only=True)
     model.load_state_dict(load_net, strict=True)
     model.eval()
     model = model.to(device)

extras/ip_adapter.py

@@ -104,7 +104,7 @@ def load_ip_adapter(clip_vision_path, ip_negative_path, ip_adapter_path):
     offload_device = torch.device('cpu')
 
     use_fp16 = model_management.should_use_fp16(device=load_device)
-    ip_state_dict = torch.load(ip_adapter_path, map_location="cpu")
+    ip_state_dict = torch.load(ip_adapter_path, map_location="cpu", weights_only=True)
     plus = "latents" in ip_state_dict["image_proj"]
     cross_attention_dim = ip_state_dict["ip_adapter"]["1.to_k_ip.weight"].shape[1]
     sdxl = cross_attention_dim == 2048

fooocus_version.py

@@ -1 +1 @@
-version = '2.5.2'
+version = '2.5.5'

language/en.json

@@ -17,6 +17,7 @@
     "Content Type": "Content Type",
     "Photograph": "Photograph",
     "Art/Anime": "Art/Anime",
+    "Apply Styles": "Apply Styles",
     "Describe this Image into Prompt": "Describe this Image into Prompt",
     "Image Size and Recommended Size": "Image Size and Recommended Size",
     "Upscale or Variation:": "Upscale or Variation:",

launch.py

@@ -80,12 +80,13 @@ if args.gpu_device_id is not None:
     os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu_device_id)
     print("Set device to:", args.gpu_device_id)
 
-if args.hf_mirror is not None : 
+if args.hf_mirror is not None:
     os.environ['HF_MIRROR'] = str(args.hf_mirror)
     print("Set hf_mirror to:", args.hf_mirror)
 
 from modules import config
 from modules.hash_cache import init_cache
+
 os.environ["U2NET_HOME"] = config.path_inpaint
 
 os.environ['GRADIO_TEMP_DIR'] = config.temp_path
@@ -100,6 +101,8 @@ if config.temp_path_cleanup_on_launch:
 
 
 def download_models(default_model, previous_default_models, checkpoint_downloads, embeddings_downloads, lora_downloads, vae_downloads):
+    from modules.util import get_file_from_folder_list
+
     for file_name, url in vae_approx_filenames:
         load_file_from_url(url=url, model_dir=config.path_vae_approx, file_name=file_name)
 
@@ -114,9 +117,9 @@ def download_models(default_model, previous_default_models, checkpoint_downloads
         return default_model, checkpoint_downloads
 
     if not args.always_download_new_model:
-        if not os.path.exists(os.path.join(config.paths_checkpoints[0], default_model)):
+        if not os.path.isfile(get_file_from_folder_list(default_model, config.paths_checkpoints)):
             for alternative_model_name in previous_default_models:
-                if os.path.exists(os.path.join(config.paths_checkpoints[0], alternative_model_name)):
+                if os.path.isfile(get_file_from_folder_list(alternative_model_name, config.paths_checkpoints)):
                     print(f'You do not have [{default_model}] but you have [{alternative_model_name}].')
                     print(f'Fooocus will use [{alternative_model_name}] to avoid downloading new models, '
                           f'but you are not using the latest models.')
@@ -126,11 +129,13 @@ def download_models(default_model, previous_default_models, checkpoint_downloads
                     break
 
     for file_name, url in checkpoint_downloads.items():
-        load_file_from_url(url=url, model_dir=config.paths_checkpoints[0], file_name=file_name)
+        model_dir = os.path.dirname(get_file_from_folder_list(file_name, config.paths_checkpoints))
+        load_file_from_url(url=url, model_dir=model_dir, file_name=file_name)
     for file_name, url in embeddings_downloads.items():
         load_file_from_url(url=url, model_dir=config.path_embeddings, file_name=file_name)
     for file_name, url in lora_downloads.items():
-        load_file_from_url(url=url, model_dir=config.paths_loras[0], file_name=file_name)
+        model_dir = os.path.dirname(get_file_from_folder_list(file_name, config.paths_loras))
+        load_file_from_url(url=url, model_dir=model_dir, file_name=file_name)
     for file_name, url in vae_downloads.items():
         load_file_from_url(url=url, model_dir=config.path_vae, file_name=file_name)
 

ldm_patched/ldm/modules/encoders/noise_aug_modules.py

@@ -8,7 +8,7 @@ class CLIPEmbeddingNoiseAugmentation(ImageConcatWithNoiseAugmentation):
         if clip_stats_path is None:
             clip_mean, clip_std = torch.zeros(timestep_dim), torch.ones(timestep_dim)
         else:
-            clip_mean, clip_std = torch.load(clip_stats_path, map_location="cpu")
+            clip_mean, clip_std = torch.load(clip_stats_path, map_location="cpu", weights_only=True)
         self.register_buffer("data_mean", clip_mean[None, :], persistent=False)
         self.register_buffer("data_std", clip_std[None, :], persistent=False)
         self.time_embed = Timestep(timestep_dim)

ldm_patched/modules/sd1_clip.py

@@ -326,7 +326,7 @@ def load_embed(embedding_name, embedding_directory, embedding_size, embed_key=No
                 except:
                     embed_out = safe_load_embed_zip(embed_path)
             else:
-                embed = torch.load(embed_path, map_location="cpu")
+                embed = torch.load(embed_path, map_location="cpu", weights_only=True)
     except Exception as e:
         print(traceback.format_exc())
         print()

ldm_patched/pfn/architecture/face/codeformer.py

@@ -377,15 +377,15 @@ class VQAutoEncoder(nn.Module):
         )
 
         if model_path is not None:
-            chkpt = torch.load(model_path, map_location="cpu")
+            chkpt = torch.load(model_path, map_location="cpu", weights_only=True)
             if "params_ema" in chkpt:
                 self.load_state_dict(
-                    torch.load(model_path, map_location="cpu")["params_ema"]
+                    torch.load(model_path, map_location="cpu", weights_only=True)["params_ema"]
                 )
                 logger.info(f"vqgan is loaded from: {model_path} [params_ema]")
             elif "params" in chkpt:
                 self.load_state_dict(
-                    torch.load(model_path, map_location="cpu")["params"]
+                    torch.load(model_path, map_location="cpu", weights_only=True)["params"]
                 )
                 logger.info(f"vqgan is loaded from: {model_path} [params]")
             else:

ldm_patched/pfn/architecture/face/gfpgan_bilinear_arch.py

@@ -273,8 +273,8 @@ class GFPGANBilinear(nn.Module):
         if decoder_load_path:
             self.stylegan_decoder.load_state_dict(
                 torch.load(
-                    decoder_load_path, map_location=lambda storage, loc: storage
+                    decoder_load_path, map_location=lambda storage, loc: storage, 
-                )["params_ema"]
+                weights_only=True)["params_ema"]
             )
         # fix decoder without updating params
         if fix_decoder:

ldm_patched/pfn/architecture/face/gfpganv1_arch.py

@@ -373,8 +373,8 @@ class GFPGANv1(nn.Module):
         if decoder_load_path:
             self.stylegan_decoder.load_state_dict(
                 torch.load(
-                    decoder_load_path, map_location=lambda storage, loc: storage
+                    decoder_load_path, map_location=lambda storage, loc: storage, 
-                )["params_ema"]
+                weights_only=True)["params_ema"]
             )
         # fix decoder without updating params
         if fix_decoder:

ldm_patched/pfn/architecture/face/gfpganv1_clean_arch.py

@@ -284,8 +284,8 @@ class GFPGANv1Clean(nn.Module):
         if decoder_load_path:
             self.stylegan_decoder.load_state_dict(
                 torch.load(
-                    decoder_load_path, map_location=lambda storage, loc: storage
+                    decoder_load_path, map_location=lambda storage, loc: storage, 
-                )["params_ema"]
+                weights_only=True)["params_ema"]
             )
         # fix decoder without updating params
         if fix_decoder:

modules/async_worker.py

@@ -1223,6 +1223,8 @@ def worker():
             height, width, _ = async_task.enhance_input_image.shape
             # input image already provided, processing is skipped
             steps = 0
+            yield_result(async_task, async_task.enhance_input_image, current_progress, async_task.black_out_nsfw, False,
+                         async_task.disable_intermediate_results)
 
         all_steps = steps * async_task.image_number
 

modules/config.py

@@ -520,10 +520,14 @@ for image_count in range(default_controlnet_image_count):
     image_count += 1
     default_ip_images[image_count] = get_config_item_or_set_default(
         key=f'default_ip_image_{image_count}',
-        default_value=None,
+        default_value='None',
-        validator=lambda x: x is None or isinstance(x, str) and os.path.exists(x),
+        validator=lambda x: x == 'None' or isinstance(x, str) and os.path.exists(x),
         expected_type=str
     )
+
+    if default_ip_images[image_count] == 'None':
+        default_ip_images[image_count] = None
+
     default_ip_types[image_count] = get_config_item_or_set_default(
         key=f'default_ip_type_{image_count}',
         default_value=modules.flags.default_ip,
@@ -698,10 +702,23 @@ default_inpaint_mask_cloth_category = get_config_item_or_set_default(
 default_inpaint_mask_sam_model = get_config_item_or_set_default(
     key='default_inpaint_mask_sam_model',
     default_value='vit_b',
-    validator=lambda x: x in [y[1] for y in modules.flags.inpaint_mask_sam_model if y[1] == x],
+    validator=lambda x: x in modules.flags.inpaint_mask_sam_model,
     expected_type=str
 )
 
+default_describe_apply_prompts_checkbox = get_config_item_or_set_default(
+    key='default_describe_apply_prompts_checkbox',
+    default_value=True,
+    validator=lambda x: isinstance(x, bool),
+    expected_type=bool
+)
+default_describe_content_type = get_config_item_or_set_default(
+    key='default_describe_content_type',
+    default_value=[modules.flags.describe_type_photo],
+    validator=lambda x: all(k in modules.flags.describe_types for k in x),
+    expected_type=list
+)
+
 config_dict["default_loras"] = default_loras = default_loras[:default_max_lora_number] + [[True, 'None', 1.0] for _ in range(default_max_lora_number - len(default_loras))]
 
 # mapping config to meta parameter

modules/core.py

@@ -231,7 +231,7 @@ def get_previewer(model):
     if vae_approx_filename in VAE_approx_models:
         VAE_approx_model = VAE_approx_models[vae_approx_filename]
     else:
-        sd = torch.load(vae_approx_filename, map_location='cpu')
+        sd = torch.load(vae_approx_filename, map_location='cpu', weights_only=True)
         VAE_approx_model = VAEApprox()
         VAE_approx_model.load_state_dict(sd)
         del sd

modules/flags.py

@@ -96,6 +96,7 @@ inpaint_options = [inpaint_option_default, inpaint_option_detail, inpaint_option
 
 describe_type_photo = 'Photograph'
 describe_type_anime = 'Art/Anime'
+describe_types = [describe_type_photo, describe_type_anime]
 
 sdxl_aspect_ratios = [
     '704*1408', '704*1344', '768*1344', '768*1280', '832*1216', '832*1152',

modules/inpaint_worker.py

@@ -196,7 +196,7 @@ class InpaintWorker:
 
         if inpaint_head_model is None:
             inpaint_head_model = InpaintHead()
-            sd = torch.load(inpaint_head_model_path, map_location='cpu')
+            sd = torch.load(inpaint_head_model_path, map_location='cpu', weights_only=True)
             inpaint_head_model.load_state_dict(sd)
 
         feed = torch.cat([

modules/upscaler.py

@@ -17,7 +17,7 @@ def perform_upscale(img):
 
     if model is None:
         model_filename = downloading_upscale_model()
-        sd = torch.load(model_filename)
+        sd = torch.load(model_filename, weights_only=True)
         sdo = OrderedDict()
         for k, v in sd.items():
             sdo[k.replace('residual_block_', 'RDB')] = v

readme.md

@@ -119,7 +119,7 @@ See also the common problems and troubleshoots [here](troubleshoot.md).
 
 ### Colab
 
-(Last tested - 2024 Mar 18 by [mashb1t](https://github.com/mashb1t))
+(Last tested - 2024 Aug 12 by [mashb1t](https://github.com/mashb1t))
 
 | Colab | Info
 | --- | --- |

update_log.md

@@ -1,3 +1,18 @@
+# [2.5.5](https://github.com/lllyasviel/Fooocus/releases/tag/v2.5.5)
+
+* Fix colab inpaint issue by moving an import statement
+
+# [2.5.4](https://github.com/lllyasviel/Fooocus/releases/tag/v2.5.4)
+
+* Fix validation for default_ip_image_* and default_inpaint_mask_sam_model
+* Fix enhance mask debugging in combination with image sorting
+* Fix loading of checkpoints and LoRAs when using multiple directories in config and then switching presets
+
+# [2.5.3](https://github.com/lllyasviel/Fooocus/releases/tag/v2.5.3)
+
+* Only load weights from non-safetensors files, preventing harmful code injection
+* Add checkbox for applying/resetting styles when describing images, also allowing multiple describe content types
+
 # [2.5.2](https://github.com/lllyasviel/Fooocus/releases/tag/v2.5.2)
 
 * Fix not adding positive prompt when styles didn't have a {prompt} placeholder in the positive prompt

webui.py

@@ -337,10 +337,11 @@ with shared.gradio_root:
                             with gr.Column():
                                 describe_input_image = grh.Image(label='Image', source='upload', type='numpy', show_label=False)
                             with gr.Column():
-                                describe_method = gr.Radio(
+                                describe_methods = gr.CheckboxGroup(
                                     label='Content Type',
-                                    choices=[flags.describe_type_photo, flags.describe_type_anime],
+                                    choices=flags.describe_types,
-                                    value=flags.describe_type_photo)
+                                    value=modules.config.default_describe_content_type)
+                                describe_apply_styles = gr.Checkbox(label='Apply Styles', value=modules.config.default_describe_apply_prompts_checkbox)
                                 describe_btn = gr.Button(value='Describe this Image into Prompt')
                                 describe_image_size = gr.Textbox(label='Image Size and Recommended Size', elem_id='describe_image_size', visible=False)
                                 gr.HTML('<a href="https://github.com/lllyasviel/Fooocus/discussions/1363" target="_blank">\U0001F4D4 Documentation</a>')
@@ -1060,30 +1061,54 @@ with shared.gradio_root:
                 gr.Audio(interactive=False, value=notification_file, elem_id='audio_notification', visible=False)
                 break
 
-        def trigger_describe(mode, img):
+        def trigger_describe(modes, img, apply_styles):
-            if mode == flags.describe_type_photo:
+            describe_prompts = []
-                from extras.interrogate import default_interrogator as default_interrogator_photo
+            styles = set()
-                return default_interrogator_photo(img), ["Fooocus V2", "Fooocus Enhance", "Fooocus Sharp"]
-            if mode == flags.describe_type_anime:
-                from extras.wd14tagger import default_interrogator as default_interrogator_anime
-                return default_interrogator_anime(img), ["Fooocus V2", "Fooocus Masterpiece"]
-            return mode, ["Fooocus V2"]
 
-        describe_btn.click(trigger_describe, inputs=[describe_method, describe_input_image],
+            if flags.describe_type_photo in modes:
-                           outputs=[prompt, style_selections], show_progress=True, queue=True)
+                from extras.interrogate import default_interrogator as default_interrogator_photo
+                describe_prompts.append(default_interrogator_photo(img))
+                styles.update(["Fooocus V2", "Fooocus Enhance", "Fooocus Sharp"])
+
+            if flags.describe_type_anime in modes:
+                from extras.wd14tagger import default_interrogator as default_interrogator_anime
+                describe_prompts.append(default_interrogator_anime(img))
+                styles.update(["Fooocus V2", "Fooocus Masterpiece"])
+
+            if len(styles) == 0 or not apply_styles:
+                styles = gr.update()
+            else:
+                styles = list(styles)
+
+            if len(describe_prompts) == 0:
+                describe_prompt = gr.update()
+            else:
+                describe_prompt = ', '.join(describe_prompts)
+
+            return describe_prompt, styles
+
+        describe_btn.click(trigger_describe, inputs=[describe_methods, describe_input_image, describe_apply_styles],
+                           outputs=[prompt, style_selections], show_progress=True, queue=True) \
+            .then(fn=style_sorter.sort_styles, inputs=style_selections, outputs=style_selections, queue=False, show_progress=False) \
+            .then(lambda: None, _js='()=>{refresh_style_localization();}')
 
         if args_manager.args.enable_auto_describe_image:
-            def trigger_auto_describe(mode, img, prompt):
+            def trigger_auto_describe(mode, img, prompt, apply_styles):
                 # keep prompt if not empty
                 if prompt == '':
-                    return trigger_describe(mode, img)
+                    return trigger_describe(mode, img, apply_styles)
                 return gr.update(), gr.update()
 
-            uov_input_image.upload(trigger_auto_describe, inputs=[describe_method, uov_input_image, prompt],
+            uov_input_image.upload(trigger_auto_describe, inputs=[describe_methods, uov_input_image, prompt, describe_apply_styles],
-                                   outputs=[prompt, style_selections], show_progress=True, queue=True)
+                                   outputs=[prompt, style_selections], show_progress=True, queue=True) \
+                .then(fn=style_sorter.sort_styles, inputs=style_selections, outputs=style_selections, queue=False, show_progress=False) \
+                .then(lambda: None, _js='()=>{refresh_style_localization();}')
 
             enhance_input_image.upload(lambda: gr.update(value=True), outputs=enhance_checkbox, queue=False, show_progress=False) \
-                .then(trigger_auto_describe, inputs=[describe_method, enhance_input_image, prompt], outputs=[prompt, style_selections], show_progress=True, queue=True)
+                .then(trigger_auto_describe, inputs=[describe_methods, enhance_input_image, prompt, describe_apply_styles],
+                      outputs=[prompt, style_selections], show_progress=True, queue=True) \
+                .then(fn=style_sorter.sort_styles, inputs=style_selections, outputs=style_selections, queue=False, show_progress=False) \
+                .then(lambda: None, _js='()=>{refresh_style_localization();}')
 
 def dump_default_english_config():
     from modules.localization import dump_english_config