feat: remove hyper-sd8 performance

still waiting for the release of hyper-sd 4step CFG LoRA, not yet satisfied with any of the CFG LoRAs compared to non-cfg ones.
see https://huggingface.co/ByteDance/Hyper-SD

language/en.json

@@ -339,8 +339,6 @@
     "sgm_uniform": "sgm_uniform",
     "simple": "simple",
     "ddim_uniform": "ddim_uniform",
-    "VAE": "VAE",
-    "Default (model)": "Default (model)",
     "Forced Overwrite of Sampling Step": "Forced Overwrite of Sampling Step",
     "Set as -1 to disable. For developer debugging.": "Set as -1 to disable. For developer debugging.",
     "Forced Overwrite of Refiner Switch Step": "Forced Overwrite of Refiner Switch Step",

ldm_patched/modules/sd.py

@@ -427,13 +427,12 @@ def load_checkpoint(config_path=None, ckpt_path=None, output_vae=True, output_cl
 
     return (ldm_patched.modules.model_patcher.ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=offload_device), clip, vae)
 
-def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, output_clipvision=False, embedding_directory=None, output_model=True, vae_filename_param=None):
+def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, output_clipvision=False, embedding_directory=None, output_model=True):
     sd = ldm_patched.modules.utils.load_torch_file(ckpt_path)
     sd_keys = sd.keys()
     clip = None
     clipvision = None
     vae = None
-    vae_filename = None
     model = None
     model_patcher = None
     clip_target = None
@@ -463,12 +462,8 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, o
         model.load_model_weights(sd, "model.diffusion_model.")
 
     if output_vae:
-        if vae_filename_param is None:
         vae_sd = ldm_patched.modules.utils.state_dict_prefix_replace(sd, {"first_stage_model.": ""}, filter_keys=True)
         vae_sd = model_config.process_vae_state_dict(vae_sd)
-        else:
-            vae_sd = ldm_patched.modules.utils.load_torch_file(vae_filename_param)
-            vae_filename = vae_filename_param
         vae = VAE(sd=vae_sd)
 
     if output_clip:
@@ -490,7 +485,7 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, o
             print("loaded straight to GPU")
             model_management.load_model_gpu(model_patcher)
 
-    return model_patcher, clip, vae, vae_filename, clipvision
+    return (model_patcher, clip, vae, clipvision)
 
 
 def load_unet_state_dict(sd): #load unet in diffusers format

modules/async_worker.py

@@ -166,7 +166,6 @@ def worker():
         adaptive_cfg = args.pop()
         sampler_name = args.pop()
         scheduler_name = args.pop()
-        vae_name = args.pop()
         overwrite_step = args.pop()
         overwrite_switch = args.pop()
         overwrite_width = args.pop()
@@ -264,6 +263,33 @@ def worker():
             adm_scaler_negative = 1.0
             adm_scaler_end = 0.0
 
+        elif performance_selection == Performance.HYPER_SD:
+            print('Enter Hyper-SD mode.')
+            progressbar(async_task, 1, 'Downloading Hyper-SD components ...')
+            loras += [(modules.config.downloading_sdxl_hyper_sd_lora(), 0.8)]
+
+            if refiner_model_name != 'None':
+                print(f'Refiner disabled in Hyper-SD mode.')
+
+            refiner_model_name = 'None'
+            sampler_name = 'dpmpp_sde_gpu'
+            scheduler_name = 'karras'
+            sharpness = 0.0
+            guidance_scale = 1.0
+            adaptive_cfg = 1.0
+            refiner_switch = 1.0
+            adm_scaler_positive = 1.0
+            adm_scaler_negative = 1.0
+            adm_scaler_end = 0.0
+
+        elif performance_selection == Performance.HYPER_SD8:
+            print('Enter Hyper-SD8 mode.')
+            progressbar(async_task, 1, 'Downloading Hyper-SD components ...')
+            loras += [(modules.config.downloading_sdxl_hyper_sd_cfg_lora(), 0.3)]
+
+            sampler_name = 'dpmpp_sde_gpu'
+            scheduler_name = 'normal'
+
         print(f'[Parameters] Adaptive CFG = {adaptive_cfg}')
         print(f'[Parameters] Sharpness = {sharpness}')
         print(f'[Parameters] ControlNet Softness = {controlnet_softness}')
@@ -429,7 +455,7 @@ def worker():
             progressbar(async_task, 3, 'Loading models ...')
             pipeline.refresh_everything(refiner_model_name=refiner_model_name, base_model_name=base_model_name,
                                         loras=loras, base_model_additional_loras=base_model_additional_loras,
-                                        use_synthetic_refiner=use_synthetic_refiner, vae_name=vae_name)
+                                        use_synthetic_refiner=use_synthetic_refiner)
 
             progressbar(async_task, 3, 'Processing prompts ...')
             tasks = []
@@ -870,7 +896,6 @@ def worker():
 
                     d.append(('Sampler', 'sampler', sampler_name))
                     d.append(('Scheduler', 'scheduler', scheduler_name))
-                    d.append(('VAE', 'vae', vae_name))
                     d.append(('Seed', 'seed', str(task['task_seed'])))
 
                     if freeu_enabled:
@@ -885,7 +910,7 @@ def worker():
                         metadata_parser = modules.meta_parser.get_metadata_parser(metadata_scheme)
                         metadata_parser.set_data(task['log_positive_prompt'], task['positive'],
                                                  task['log_negative_prompt'], task['negative'],
-                                                 steps, base_model_name, refiner_model_name, loras, vae_name)
+                                                 steps, base_model_name, refiner_model_name, loras)
                     d.append(('Metadata Scheme', 'metadata_scheme', metadata_scheme.value if save_metadata_to_images else save_metadata_to_images))
                     d.append(('Version', 'version', 'Fooocus v' + fooocus_version.version))
                     img_paths.append(log(x, d, metadata_parser, output_format))

modules/config.py

@@ -189,7 +189,6 @@ paths_checkpoints = get_dir_or_set_default('path_checkpoints', ['../models/check
 paths_loras = get_dir_or_set_default('path_loras', ['../models/loras/'], True)
 path_embeddings = get_dir_or_set_default('path_embeddings', '../models/embeddings/')
 path_vae_approx = get_dir_or_set_default('path_vae_approx', '../models/vae_approx/')
-path_vae = get_dir_or_set_default('path_vae', '../models/vae/')
 path_upscale_models = get_dir_or_set_default('path_upscale_models', '../models/upscale_models/')
 path_inpaint = get_dir_or_set_default('path_inpaint', '../models/inpaint/')
 path_controlnet = get_dir_or_set_default('path_controlnet', '../models/controlnet/')
@@ -347,11 +346,6 @@ default_scheduler = get_config_item_or_set_default(
     default_value='karras',
     validator=lambda x: x in modules.flags.scheduler_list
 )
-default_vae = get_config_item_or_set_default(
-    key='default_vae',
-    default_value=modules.flags.default_vae,
-    validator=lambda x: isinstance(x, str)
-)
 default_styles = get_config_item_or_set_default(
     key='default_styles',
     default_value=[
@@ -541,32 +535,27 @@ with open(config_example_path, "w", encoding="utf-8") as json_file:
 
 model_filenames = []
 lora_filenames = []
-vae_filenames = []
 wildcard_filenames = []
 
 sdxl_lcm_lora = 'sdxl_lcm_lora.safetensors'
 sdxl_lightning_lora = 'sdxl_lightning_4step_lora.safetensors'
-loras_metadata_remove = [sdxl_lcm_lora, sdxl_lightning_lora]
+sdxl_hyper_sd_lora = 'sdxl_hyper_sd_4step_lora.safetensors'
+loras_metadata_remove = [sdxl_lcm_lora, sdxl_lightning_lora, sdxl_hyper_sd_lora]
 
 
 def get_model_filenames(folder_paths, extensions=None, name_filter=None):
     if extensions is None:
         extensions = ['.pth', '.ckpt', '.bin', '.safetensors', '.fooocus.patch']
     files = []
-
-    if not isinstance(folder_paths, list):
-        folder_paths = [folder_paths]
     for folder in folder_paths:
         files += get_files_from_folder(folder, extensions, name_filter)
-
     return files
 
 
 def update_files():
-    global model_filenames, lora_filenames, vae_filenames, wildcard_filenames, available_presets
+    global model_filenames, lora_filenames, wildcard_filenames, available_presets
     model_filenames = get_model_filenames(paths_checkpoints)
     lora_filenames = get_model_filenames(paths_loras)
-    vae_filenames = get_model_filenames(path_vae)
     wildcard_filenames = get_files_from_folder(path_wildcards, ['.txt'])
     available_presets = get_presets()
     return
@@ -620,13 +609,22 @@ def downloading_sdxl_lcm_lora():
 
 def downloading_sdxl_lightning_lora():
     load_file_from_url(
-        url='https://huggingface.co/ByteDance/SDXL-Lightning/resolve/main/sdxl_lightning_4step_lora.safetensors',
+        url='https://huggingface.co/mashb1t/misc/resolve/main/sdxl_lightning_4step_lora.safetensors',
         model_dir=paths_loras[0],
         file_name=sdxl_lightning_lora
     )
     return sdxl_lightning_lora
 
 
+def downloading_sdxl_hyper_sd_lora():
+    load_file_from_url(
+        url='https://huggingface.co/mashb1t/misc/resolve/main/sdxl_hyper_sd_4step_lora.safetensors',
+        model_dir=paths_loras[0],
+        file_name=sdxl_hyper_sd_lora
+    )
+    return sdxl_hyper_sd_lora
+
+
 def downloading_controlnet_canny():
     load_file_from_url(
         url='https://huggingface.co/lllyasviel/misc/resolve/main/control-lora-canny-rank128.safetensors',

modules/core.py

@@ -35,13 +35,12 @@ opModelSamplingDiscrete = ModelSamplingDiscrete()
 
 
 class StableDiffusionModel:
-    def __init__(self, unet=None, vae=None, clip=None, clip_vision=None, filename=None, vae_filename=None):
+    def __init__(self, unet=None, vae=None, clip=None, clip_vision=None, filename=None):
         self.unet = unet
         self.vae = vae
         self.clip = clip
         self.clip_vision = clip_vision
         self.filename = filename
-        self.vae_filename = vae_filename
         self.unet_with_lora = unet
         self.clip_with_lora = clip
         self.visited_loras = ''
@@ -143,10 +142,9 @@ def apply_controlnet(positive, negative, control_net, image, strength, start_per
 
 @torch.no_grad()
 @torch.inference_mode()
-def load_model(ckpt_filename, vae_filename=None):
+def load_model(ckpt_filename):
-    unet, clip, vae, vae_filename, clip_vision = load_checkpoint_guess_config(ckpt_filename, embedding_directory=path_embeddings,
+    unet, clip, vae, clip_vision = load_checkpoint_guess_config(ckpt_filename, embedding_directory=path_embeddings)
-                                                                vae_filename_param=vae_filename)
+    return StableDiffusionModel(unet=unet, clip=clip, vae=vae, clip_vision=clip_vision, filename=ckpt_filename)
-    return StableDiffusionModel(unet=unet, clip=clip, vae=vae, clip_vision=clip_vision, filename=ckpt_filename, vae_filename=vae_filename)
 
 
 @torch.no_grad()

modules/default_pipeline.py

@@ -3,7 +3,6 @@ import os
 import torch
 import modules.patch
 import modules.config
-import modules.flags
 import ldm_patched.modules.model_management
 import ldm_patched.modules.latent_formats
 import modules.inpaint_worker
@@ -59,21 +58,17 @@ def assert_model_integrity():
 
 @torch.no_grad()
 @torch.inference_mode()
-def refresh_base_model(name, vae_name=None):
+def refresh_base_model(name):
     global model_base
 
     filename = get_file_from_folder_list(name, modules.config.paths_checkpoints)
 
-    vae_filename = None
+    if model_base.filename == filename:
-    if vae_name is not None and vae_name != modules.flags.default_vae:
-        vae_filename = get_file_from_folder_list(vae_name, modules.config.path_vae)
-
-    if model_base.filename == filename and model_base.vae_filename == vae_filename:
         return
 
-    model_base = core.load_model(filename, vae_filename)
+    model_base = core.StableDiffusionModel()
+    model_base = core.load_model(filename)
     print(f'Base model loaded: {model_base.filename}')
-    print(f'VAE loaded: {model_base.vae_filename}')
     return
 
 
@@ -221,7 +216,7 @@ def prepare_text_encoder(async_call=True):
 @torch.no_grad()
 @torch.inference_mode()
 def refresh_everything(refiner_model_name, base_model_name, loras,
-                       base_model_additional_loras=None, use_synthetic_refiner=False, vae_name=None):
+                       base_model_additional_loras=None, use_synthetic_refiner=False):
     global final_unet, final_clip, final_vae, final_refiner_unet, final_refiner_vae, final_expansion
 
     final_unet = None
@@ -232,11 +227,11 @@ def refresh_everything(refiner_model_name, base_model_name, loras,
 
     if use_synthetic_refiner and refiner_model_name == 'None':
         print('Synthetic Refiner Activated')
-        refresh_base_model(base_model_name, vae_name)
+        refresh_base_model(base_model_name)
         synthesize_refiner_model()
     else:
         refresh_refiner_model(refiner_model_name)
-        refresh_base_model(base_model_name, vae_name)
+        refresh_base_model(base_model_name)
 
     refresh_loras(loras, base_model_additional_loras=base_model_additional_loras)
     assert_model_integrity()
@@ -259,8 +254,7 @@ def refresh_everything(refiner_model_name, base_model_name, loras,
 refresh_everything(
     refiner_model_name=modules.config.default_refiner_model_name,
     base_model_name=modules.config.default_base_model_name,
-    loras=get_enabled_loras(modules.config.default_loras),
+    loras=get_enabled_loras(modules.config.default_loras)
-    vae_name=modules.config.default_vae,
 )
 
 

modules/flags.py

@@ -53,8 +53,6 @@ SAMPLER_NAMES = KSAMPLER_NAMES + list(SAMPLER_EXTRA.keys())
 sampler_list = SAMPLER_NAMES
 scheduler_list = SCHEDULER_NAMES
 
-default_vae = 'Default (model)'
-
 refiner_swap_method = 'joint'
 
 cn_ip = "ImagePrompt"
@@ -109,6 +107,7 @@ class Steps(IntEnum):
     SPEED = 30
     EXTREME_SPEED = 8
     LIGHTNING = 4
+    HYPER_SD = 4
 
 
 class StepsUOV(IntEnum):
@@ -116,6 +115,7 @@ class StepsUOV(IntEnum):
     SPEED = 18
     EXTREME_SPEED = 8
     LIGHTNING = 4
+    HYPER_SD = 4
 
 
 class Performance(Enum):
@@ -123,6 +123,7 @@ class Performance(Enum):
     SPEED = 'Speed'
     EXTREME_SPEED = 'Extreme Speed'
     LIGHTNING = 'Lightning'
+    HYPER_SD = 'Hyper-SD'
 
     @classmethod
     def list(cls) -> list:
@@ -132,7 +133,7 @@ class Performance(Enum):
     def has_restricted_features(cls, x) -> bool:
         if isinstance(x, Performance):
             x = x.value
-        return x in [cls.EXTREME_SPEED.value, cls.LIGHTNING.value]
+        return x in [cls.EXTREME_SPEED.value, cls.LIGHTNING.value, cls.HYPER_SD.value]
 
     def steps(self) -> int | None:
         return Steps[self.name].value if Steps[self.name] else None

modules/meta_parser.py

@@ -46,7 +46,6 @@ def load_parameter_button_click(raw_metadata: dict | str, is_generating: bool):
     get_float('refiner_switch', 'Refiner Switch', loaded_parameter_dict, results)
     get_str('sampler', 'Sampler', loaded_parameter_dict, results)
     get_str('scheduler', 'Scheduler', loaded_parameter_dict, results)
-    get_str('vae', 'VAE', loaded_parameter_dict, results)
     get_seed('seed', 'Seed', loaded_parameter_dict, results)
 
     if is_generating:
@@ -254,7 +253,6 @@ class MetadataParser(ABC):
         self.refiner_model_name: str = ''
         self.refiner_model_hash: str = ''
         self.loras: list = []
-        self.vae_name: str = ''
 
     @abstractmethod
     def get_scheme(self) -> MetadataScheme:
@@ -269,7 +267,7 @@ class MetadataParser(ABC):
         raise NotImplementedError
 
     def set_data(self, raw_prompt, full_prompt, raw_negative_prompt, full_negative_prompt, steps, base_model_name,
-                 refiner_model_name, loras, vae_name):
+                 refiner_model_name, loras):
         self.raw_prompt = raw_prompt
         self.full_prompt = full_prompt
         self.raw_negative_prompt = raw_negative_prompt
@@ -291,7 +289,6 @@ class MetadataParser(ABC):
                 lora_path = get_file_from_folder_list(lora_name, modules.config.paths_loras)
                 lora_hash = get_sha256(lora_path)
                 self.loras.append((Path(lora_name).stem, lora_weight, lora_hash))
-        self.vae_name = Path(vae_name).stem
 
     @staticmethod
     def remove_special_loras(lora_filenames):
@@ -313,7 +310,6 @@ class A1111MetadataParser(MetadataParser):
         'steps': 'Steps',
         'sampler': 'Sampler',
         'scheduler': 'Scheduler',
-        'vae': 'VAE',
         'guidance_scale': 'CFG scale',
         'seed': 'Seed',
         'resolution': 'Size',
@@ -401,12 +397,13 @@ class A1111MetadataParser(MetadataParser):
                     data['sampler'] = k
                     break
 
-        for key in ['base_model', 'refiner_model', 'vae']:
+        for key in ['base_model', 'refiner_model']:
             if key in data:
-                if key == 'vae':
+                for filename in modules.config.model_filenames:
-                    self.add_extension_to_filename(data, modules.config.vae_filenames, 'vae')
+                    path = Path(filename)
-                else:
+                    if data[key] == path.stem:
-                    self.add_extension_to_filename(data, modules.config.model_filenames, key)
+                        data[key] = filename
+                        break
 
         lora_data = ''
         if 'lora_weights' in data and data['lora_weights'] != '':
@@ -436,7 +433,6 @@ class A1111MetadataParser(MetadataParser):
 
         sampler = data['sampler']
         scheduler = data['scheduler']
-
         if sampler in SAMPLERS and SAMPLERS[sampler] != '':
             sampler = SAMPLERS[sampler]
             if sampler not in CIVITAI_NO_KARRAS and scheduler == 'karras':
@@ -455,7 +451,6 @@ class A1111MetadataParser(MetadataParser):
 
             self.fooocus_to_a1111['performance']: data['performance'],
             self.fooocus_to_a1111['scheduler']: scheduler,
-            self.fooocus_to_a1111['vae']: Path(data['vae']).stem,
             # workaround for multiline prompts
             self.fooocus_to_a1111['raw_prompt']: self.raw_prompt,
             self.fooocus_to_a1111['raw_negative_prompt']: self.raw_negative_prompt,
@@ -496,14 +491,6 @@ class A1111MetadataParser(MetadataParser):
         negative_prompt_text = f"\nNegative prompt: {negative_prompt_resolved}" if negative_prompt_resolved else ""
         return f"{positive_prompt_resolved}{negative_prompt_text}\n{generation_params_text}".strip()
 
-    @staticmethod
-    def add_extension_to_filename(data, filenames, key):
-        for filename in filenames:
-            path = Path(filename)
-            if data[key] == path.stem:
-                data[key] = filename
-                break
-
 
 class FooocusMetadataParser(MetadataParser):
     def get_scheme(self) -> MetadataScheme:
@@ -512,7 +499,6 @@ class FooocusMetadataParser(MetadataParser):
     def parse_json(self, metadata: dict) -> dict:
         model_filenames = modules.config.model_filenames.copy()
         lora_filenames = modules.config.lora_filenames.copy()
-        vae_filenames = modules.config.vae_filenames.copy()
         self.remove_special_loras(lora_filenames)
         for key, value in metadata.items():
             if value in ['', 'None']:
@@ -521,8 +507,6 @@ class FooocusMetadataParser(MetadataParser):
                 metadata[key] = self.replace_value_with_filename(key, value, model_filenames)
             elif key.startswith('lora_combined_'):
                 metadata[key] = self.replace_value_with_filename(key, value, lora_filenames)
-            elif key == 'vae':
-                metadata[key] = self.replace_value_with_filename(key, value, vae_filenames)
             else:
                 continue
 
@@ -549,7 +533,6 @@ class FooocusMetadataParser(MetadataParser):
             res['refiner_model'] = self.refiner_model_name
             res['refiner_model_hash'] = self.refiner_model_hash
 
-        res['vae'] = self.vae_name
         res['loras'] = self.loras
 
         if modules.config.metadata_created_by != '':

modules/util.py

@@ -371,9 +371,6 @@ def is_json(data: str) -> bool:
 
 
 def get_file_from_folder_list(name, folders):
-    if not isinstance(folders, list):
-        folders = [folders]
-
     for folder in folders:
         filename = os.path.abspath(os.path.realpath(os.path.join(folder, name)))
         if os.path.isfile(filename):

webui.py

@@ -406,8 +406,6 @@ with shared.gradio_root:
                                                    value=modules.config.default_sampler)
                         scheduler_name = gr.Dropdown(label='Scheduler', choices=flags.scheduler_list,
                                                      value=modules.config.default_scheduler)
-                        vae_name = gr.Dropdown(label='VAE', choices=[modules.flags.default_vae] + modules.config.vae_filenames,
-                                                     value=modules.config.default_vae, show_label=True)
 
                         generate_image_grid = gr.Checkbox(label='Generate Image Grid for Each Batch',
                                                           info='(Experimental) This may cause performance problems on some computers and certain internet conditions.',
@@ -530,7 +528,6 @@ with shared.gradio_root:
                     modules.config.update_files()
                     results = [gr.update(choices=modules.config.model_filenames)]
                     results += [gr.update(choices=['None'] + modules.config.model_filenames)]
-                    results += [gr.update(choices=['None'] + modules.config.vae_filenames)]
                     if not args_manager.args.disable_preset_selection:
                         results += [gr.update(choices=modules.config.available_presets)]
                     for i in range(modules.config.default_max_lora_number):
@@ -538,7 +535,7 @@ with shared.gradio_root:
                                     gr.update(choices=['None'] + modules.config.lora_filenames), gr.update()]
                     return results
 
-                refresh_files_output = [base_model, refiner_model, vae_name]
+                refresh_files_output = [base_model, refiner_model]
                 if not args_manager.args.disable_preset_selection:
                     refresh_files_output += [preset_selection]
                 refresh_files.click(refresh_files_clicked, [], refresh_files_output + lora_ctrls,
@@ -550,8 +547,8 @@ with shared.gradio_root:
                              performance_selection, overwrite_step, overwrite_switch, aspect_ratios_selection,
                              overwrite_width, overwrite_height, guidance_scale, sharpness, adm_scaler_positive,
                              adm_scaler_negative, adm_scaler_end, refiner_swap_method, adaptive_cfg, base_model,
-                             refiner_model, refiner_switch, sampler_name, scheduler_name, vae_name, seed_random,
+                             refiner_model, refiner_switch, sampler_name, scheduler_name, seed_random, image_seed,
-                             image_seed, generate_button, load_parameter_button] + freeu_ctrls + lora_ctrls
+                             generate_button, load_parameter_button] + freeu_ctrls + lora_ctrls
 
         if not args_manager.args.disable_preset_selection:
             def preset_selection_change(preset, is_generating):
@@ -637,7 +634,7 @@ with shared.gradio_root:
         ctrls += [outpaint_selections, inpaint_input_image, inpaint_additional_prompt, inpaint_mask_image]
         ctrls += [disable_preview, disable_intermediate_results, disable_seed_increment]
         ctrls += [adm_scaler_positive, adm_scaler_negative, adm_scaler_end, adaptive_cfg]
-        ctrls += [sampler_name, scheduler_name, vae_name]
+        ctrls += [sampler_name, scheduler_name]
         ctrls += [overwrite_step, overwrite_switch, overwrite_width, overwrite_height, overwrite_vary_strength]
         ctrls += [overwrite_upscale_strength, mixing_image_prompt_and_vary_upscale, mixing_image_prompt_and_inpaint]
         ctrls += [debugging_cn_preprocessor, skipping_cn_preprocessor, canny_low_threshold, canny_high_threshold]