release: bump version to 2.4.1, update changelog (#3027)

* feat: build docker image on push to main branch

* feat: add tag "edge" for main when building the docker image

* feat: update name of build container workflow

.dockerignore

@@ -1 +1,54 @@
-.idea
+__pycache__
+*.ckpt
+*.safetensors
+*.pth
+*.pt
+*.bin
+*.patch
+*.backup
+*.corrupted
+*.partial
+*.onnx
+sorted_styles.json
+/input
+/cache
+/language/default.json
+/test_imgs
+config.txt
+config_modification_tutorial.txt
+user_path_config.txt
+user_path_config-deprecated.txt
+/modules/*.png
+/repositories
+/fooocus_env
+/venv
+/tmp
+/ui-config.json
+/outputs
+/config.json
+/log
+/webui.settings.bat
+/embeddings
+/styles.csv
+/params.txt
+/styles.csv.bak
+/webui-user.bat
+/webui-user.sh
+/interrogate
+/user.css
+/.idea
+/notification.ogg
+/notification.mp3
+/SwinIR
+/textual_inversion
+.vscode
+/extensions
+/test/stdout.txt
+/test/stderr.txt
+/cache.json*
+/config_states/
+/node_modules
+/package-lock.json
+/.coverage*
+/auth.json
+.DS_Store

.gitattributes

@@ -0,0 +1,3 @@
+# Ensure that shell scripts always use lf line endings, e.g. entrypoint.sh for docker
+* text=auto
+*.sh text eol=lf

.github/CODEOWNERS

@@ -1 +1 @@
-*       @lllyasviel
+*       @mashb1t

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"

.github/workflows/build_container.yml

@@ -0,0 +1,47 @@
+name: Docker image build
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - v*
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=edge,branch=main
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

Dockerfile

@@ -1,4 +1,4 @@
-FROM nvidia/cuda:12.3.1-base-ubuntu22.04
+FROM nvidia/cuda:12.4.1-base-ubuntu22.04
 ENV DEBIAN_FRONTEND noninteractive
 ENV CMDARGS --listen
 
@@ -23,7 +23,7 @@ RUN chown -R user:user /content
 WORKDIR /content
 USER user
 
-RUN git clone https://github.com/lllyasviel/Fooocus /content/app
+COPY . /content/app
 RUN mv /content/app/models /content/app/models.org
 
 CMD [ "sh", "-c", "/content/entrypoint.sh ${CMDARGS}" ]

css/style.css

@@ -27,6 +27,7 @@ progress {
   border-radius: 5px; /* Round the corners of the progress bar */
   background-color: #f3f3f3; /* Light grey background */
   width: 100%;
+  vertical-align: middle !important;
 }
 
 /* Style the progress bar container */
@@ -69,6 +70,11 @@ progress::after {
   height: 30px !important;
 }
 
+.progress-bar span {
+    text-align: right;
+    width: 215px;
+}
+
 .type_row{
   height: 80px !important;
 }
@@ -101,10 +107,14 @@ progress::after {
   overflow: auto !important;
 }
 
-.aspect_ratios label {
+.performance_selection label {
     width: 140px !important;
 }
 
+.aspect_ratios label {
+    flex: calc(50% - 5px) !important;
+}
+
 .aspect_ratios label span {
     white-space: nowrap !important;
 }
@@ -393,4 +403,12 @@ progress::after {
   text-align: center;
   border-radius: 5px 5px 0px 0px;
   display: none; /* remove this to enable tooltip in preview image */
+}
+
+#inpaint_canvas .canvas-tooltip-info {
+  top: 2px;
+}
+
+#inpaint_brush_color input[type=color]{
+  background: none;
 }

docker-compose.yml

@@ -1,12 +1,10 @@
-version: '3.9'
-
 volumes:
   fooocus-data:
 
 services:
   app:
     build: .
-    image: fooocus
+    image: ghcr.io/lllyasviel/fooocus
     ports:
      - "7865:7865"
     environment:

docker.md

@@ -1,35 +1,99 @@
 # Fooocus on Docker
 
-The docker image is based on NVIDIA CUDA 12.3 and PyTorch 2.0, see [Dockerfile](Dockerfile) and [requirements_docker.txt](requirements_docker.txt) for details.
+The docker image is based on NVIDIA CUDA 12.4 and PyTorch 2.1, see [Dockerfile](Dockerfile) and [requirements_docker.txt](requirements_docker.txt) for details.
+
+## Requirements
+
+- A computer with specs good enough to run Fooocus, and proprietary Nvidia drivers
+- Docker, Docker Compose, or Podman
 
 ## Quick start
 
-**This is just an easy way for testing. Please find more information in the [notes](#notes).**
+**More information in the [notes](#notes).**
+
+### Running with Docker Compose
 
 1. Clone this repository
-2. Build the image with `docker compose build`
+2. Run the docker container with `docker compose up`.
-3. Run the docker container with `docker compose up`. Building the image takes some time.
+
+### Running with Docker
+
+```sh
+docker run -p 7865:7865 -v fooocus-data:/content/data -it \
+--gpus all \
+-e CMDARGS=--listen \
+-e DATADIR=/content/data \
+-e config_path=/content/data/config.txt \
+-e config_example_path=/content/data/config_modification_tutorial.txt \
+-e path_checkpoints=/content/data/models/checkpoints/ \
+-e path_loras=/content/data/models/loras/ \
+-e path_embeddings=/content/data/models/embeddings/ \
+-e path_vae_approx=/content/data/models/vae_approx/ \
+-e path_upscale_models=/content/data/models/upscale_models/ \
+-e path_inpaint=/content/data/models/inpaint/ \
+-e path_controlnet=/content/data/models/controlnet/ \
+-e path_clip_vision=/content/data/models/clip_vision/ \
+-e path_fooocus_expansion=/content/data/models/prompt_expansion/fooocus_expansion/ \
+-e path_outputs=/content/app/outputs/ \
+ghcr.io/lllyasviel/fooocus
+```
+### Running with Podman
+
+```sh
+podman run -p 7865:7865 -v fooocus-data:/content/data -it \
+--security-opt=no-new-privileges --cap-drop=ALL --security-opt label=type:nvidia_container_t --device=nvidia.com/gpu=all \
+-e CMDARGS=--listen \
+-e DATADIR=/content/data \
+-e config_path=/content/data/config.txt \
+-e config_example_path=/content/data/config_modification_tutorial.txt \
+-e path_checkpoints=/content/data/models/checkpoints/ \
+-e path_loras=/content/data/models/loras/ \
+-e path_embeddings=/content/data/models/embeddings/ \
+-e path_vae_approx=/content/data/models/vae_approx/ \
+-e path_upscale_models=/content/data/models/upscale_models/ \
+-e path_inpaint=/content/data/models/inpaint/ \
+-e path_controlnet=/content/data/models/controlnet/ \
+-e path_clip_vision=/content/data/models/clip_vision/ \
+-e path_fooocus_expansion=/content/data/models/prompt_expansion/fooocus_expansion/ \
+-e path_outputs=/content/app/outputs/ \
+ghcr.io/lllyasviel/fooocus
+```
 
 When you see the message  `Use the app with http://0.0.0.0:7865/` in the console, you can access the URL in your browser.
 
-Your models and outputs are stored in the `fooocus-data` volume, which, depending on OS, is stored in `/var/lib/docker/volumes`.
+Your models and outputs are stored in the `fooocus-data` volume, which, depending on OS, is stored in `/var/lib/docker/volumes/` (or `~/.local/share/containers/storage/volumes/` when using `podman`).
+
+## Building the container locally
+
+Clone the repository first, and open a terminal in the folder.
+
+Build with `docker`:
+```sh
+docker build . -t fooocus
+```
+
+Build with `podman`:
+```sh
+podman build . -t fooocus
+```
 
 ## Details
 
-### Update the container manually
+### Update the container manually (`docker compose`)
 
 When you are using `docker compose up` continuously, the container is not updated to the latest version of Fooocus automatically.
 Run `git pull` before executing `docker compose build --no-cache` to build an image with the latest Fooocus version.
 You can then start it with `docker compose up`
 
 ### Import models, outputs
-If you want to import files from models or the outputs folder, you can uncomment the following settings in the [docker-compose.yml](docker-compose.yml):
+
+If you want to import files from models or the outputs folder, you can add the following bind mounts in the [docker-compose.yml](docker-compose.yml) or your preferred method of running the container:
 ```
 #- ./models:/import/models   # Once you import files, you don't need to mount again.
 #- ./outputs:/import/outputs  # Once you import files, you don't need to mount again.
 ```
-After running `docker compose up`, your files will be copied into `/content/data/models` and `/content/data/outputs`
+After running the container, your files will be copied into `/content/data/models` and `/content/data/outputs`
-Since `/content/data` is a persistent volume folder, your files will be persisted even when you re-run `docker compose up --build` without above volume settings.
+Since `/content/data` is a persistent volume folder, your files will be persisted even when you re-run the container without the above mounts.
 
 
 ### Paths inside the container

extras/censor.py

@@ -1,56 +1,60 @@
-# modified version of https://github.com/AUTOMATIC1111/stable-diffusion-webui-nsfw-censor/blob/master/scripts/censor.py
-import numpy as np
 import os
 
-from extras.safety_checker.models.safety_checker import StableDiffusionSafetyChecker
+import numpy as np
-from transformers import CLIPFeatureExtractor, CLIPConfig
+import torch
-from PIL import Image
+from transformers import CLIPConfig, CLIPImageProcessor
+
+import ldm_patched.modules.model_management as model_management
 import modules.config
+from extras.safety_checker.models.safety_checker import StableDiffusionSafetyChecker
+from ldm_patched.modules.model_patcher import ModelPatcher
 
 safety_checker_repo_root = os.path.join(os.path.dirname(__file__), 'safety_checker')
 config_path = os.path.join(safety_checker_repo_root, "configs", "config.json")
 preprocessor_config_path = os.path.join(safety_checker_repo_root, "configs", "preprocessor_config.json")
 
-safety_feature_extractor = None
+
-safety_checker = None
+class Censor:
+    def __init__(self):
+        self.safety_checker_model: ModelPatcher | None = None
+        self.clip_image_processor: CLIPImageProcessor | None = None
+        self.load_device = torch.device('cpu')
+        self.offload_device = torch.device('cpu')
+
+    def init(self):
+        if self.safety_checker_model is None and self.clip_image_processor is None:
+            safety_checker_model = modules.config.downloading_safety_checker_model()
+            self.clip_image_processor = CLIPImageProcessor.from_json_file(preprocessor_config_path)
+            clip_config = CLIPConfig.from_json_file(config_path)
+            model = StableDiffusionSafetyChecker.from_pretrained(safety_checker_model, config=clip_config)
+            model.eval()
+
+            self.load_device = model_management.text_encoder_device()
+            self.offload_device = model_management.text_encoder_offload_device()
+
+            model.to(self.offload_device)
+
+            self.safety_checker_model = ModelPatcher(model, load_device=self.load_device, offload_device=self.offload_device)
+
+    def censor(self, images: list | np.ndarray) -> list | np.ndarray:
+        self.init()
+        model_management.load_model_gpu(self.safety_checker_model)
+
+        single = False
+        if not isinstance(images, list) or isinstance(images, np.ndarray):
+            images = [images]
+            single = True
+
+        safety_checker_input = self.clip_image_processor(images, return_tensors="pt")
+        safety_checker_input.to(device=self.load_device)
+        checked_images, has_nsfw_concept = self.safety_checker_model.model(images=images,
+                                                                           clip_input=safety_checker_input.pixel_values)
+        checked_images = [image.astype(np.uint8) for image in checked_images]
+
+        if single:
+            checked_images = checked_images[0]
+
+        return checked_images
 
 
-def numpy_to_pil(image):
+default_censor = Censor().censor
-    image = (image * 255).round().astype("uint8")
-    pil_image = Image.fromarray(image)
-
-    return pil_image
-
-
-# check and replace nsfw content
-def check_safety(x_image):
-    global safety_feature_extractor, safety_checker
-
-    if safety_feature_extractor is None or safety_checker is None:
-        safety_checker_model = modules.config.downloading_safety_checker_model()
-        safety_feature_extractor = CLIPFeatureExtractor.from_json_file(preprocessor_config_path)
-        clip_config = CLIPConfig.from_json_file(config_path)
-        safety_checker = StableDiffusionSafetyChecker.from_pretrained(safety_checker_model, config=clip_config)
-
-    safety_checker_input = safety_feature_extractor(numpy_to_pil(x_image), return_tensors="pt")
-    x_checked_image, has_nsfw_concept = safety_checker(images=x_image, clip_input=safety_checker_input.pixel_values)
-
-    return x_checked_image, has_nsfw_concept
-
-
-def censor_single(x):
-    x_checked_image, has_nsfw_concept = check_safety(x)
-
-    # replace image with black pixels, keep dimensions
-    # workaround due to different numpy / pytorch image matrix format
-    if has_nsfw_concept[0]:
-        imageshape = x_checked_image.shape
-        x_checked_image = np.zeros((imageshape[0], imageshape[1], 3), dtype = np.uint8)
-
-    return x_checked_image
-
-
-def censor_batch(images):
-    images = [censor_single(image) for image in images]
-
-    return images

fooocus_version.py

@@ -1 +1 @@
-version = '2.4.0-rc1'
+version = '2.4.1'

javascript/localization.js

@@ -80,6 +80,15 @@ function refresh_style_localization() {
     processNode(document.querySelector('.style_selections'));
 }
 
+function refresh_aspect_ratios_label(value) {
+    label = document.querySelector('#aspect_ratios_accordion div span[data-original-text="Aspect Ratios"]');
+    translation = getTranslation("Aspect Ratios");
+    if (typeof translation == "undefined") {
+        translation = "Aspect Ratios";
+    }
+    label.textContent = translation + " " + htmlDecode(value);
+}
+
 function localizeWholePage() {
     processNode(gradioApp());
 

javascript/script.js

@@ -256,3 +256,8 @@ function set_theme(theme) {
         window.location.replace(gradioURL + '?__theme=' + theme);
     }
 }
+
+function htmlDecode(input) {
+  var doc = new DOMParser().parseFromString(input, "text/html");
+  return doc.documentElement.textContent;
+}

language/en.json

@@ -9,8 +9,15 @@
     "Advanced": "Advanced",
     "Upscale or Variation": "Upscale or Variation",
     "Image Prompt": "Image Prompt",
-    "Inpaint or Outpaint (beta)": "Inpaint or Outpaint (beta)",
+    "Inpaint or Outpaint": "Inpaint or Outpaint",
-    "Drag above image to here": "Drag above image to here",
+    "Outpaint Direction": "Outpaint Direction",
+    "Method": "Method",
+    "Describe": "Describe",
+    "Content Type": "Content Type",
+    "Photograph": "Photograph",
+    "Art/Anime": "Art/Anime",
+    "Describe this Image into Prompt": "Describe this Image into Prompt",
+    "Image Size and Recommended Size": "Image Size and Recommended Size",
     "Upscale or Variation:": "Upscale or Variation:",
     "Disabled": "Disabled",
     "Vary (Subtle)": "Vary (Subtle)",
@@ -313,6 +320,8 @@
     "vae": "vae",
     "CFG Mimicking from TSNR": "CFG Mimicking from TSNR",
     "Enabling Fooocus's implementation of CFG mimicking for TSNR (effective when real CFG > mimicked CFG).": "Enabling Fooocus's implementation of CFG mimicking for TSNR (effective when real CFG > mimicked CFG).",
+    "CLIP Skip": "CLIP Skip",
+    "Bypass CLIP layers to avoid overfitting (use 1 to not skip any layers, 2 is recommended).": "Bypass CLIP layers to avoid overfitting (use 1 to not skip any layers, 2 is recommended).",
     "Sampler": "Sampler",
     "dpmpp_2m_sde_gpu": "dpmpp_2m_sde_gpu",
     "Only effective in non-inpaint mode.": "Only effective in non-inpaint mode.",
@@ -384,7 +393,7 @@
     "Fooocus Enhance": "Fooocus Enhance",
     "Fooocus Cinematic": "Fooocus Cinematic",
     "Fooocus Sharp": "Fooocus Sharp",
-    "Drag any image generated by Fooocus here": "Drag any image generated by Fooocus here",
+    "For images created by Fooocus": "For images created by Fooocus",
     "Metadata": "Metadata",
     "Apply Metadata": "Apply Metadata",
     "Metadata Scheme": "Metadata Scheme",

modules/async_worker.py

@@ -44,12 +44,12 @@ def worker():
     import fooocus_version
     import args_manager
 
-    from extras.censor import censor_batch, censor_single
+    from extras.censor import default_censor
     from modules.sdxl_styles import apply_style, get_random_style, fooocus_expansion, apply_arrays, random_style_name
     from modules.private_logger import log
     from extras.expansion import safe_str
     from modules.util import (remove_empty_str, HWC3, resize_image, get_image_shape_ceil, set_image_shape_ceil,
-                              get_shape_ceil, resample_image, erode_or_dilate, ordinal_suffix, get_enabled_loras,
+                              get_shape_ceil, resample_image, erode_or_dilate, get_enabled_loras,
                               parse_lora_references_from_prompt, apply_wildcards)
     from modules.upscaler import perform_upscale
     from modules.flags import Performance
@@ -72,13 +72,13 @@ def worker():
         async_task.yields.append(['preview', (number, text, None)])
 
     def yield_result(async_task, imgs, black_out_nsfw, censor=True, do_not_show_finished_images=False,
-                     progressbar_index=13):
+                     progressbar_index=flags.preparation_step_count):
         if not isinstance(imgs, list):
             imgs = [imgs]
 
         if censor and (modules.config.default_black_out_nsfw or black_out_nsfw):
             progressbar(async_task, progressbar_index, 'Checking for NSFW content ...')
-            imgs = censor_batch(imgs)
+            imgs = default_censor(imgs)
 
         async_task.results = async_task.results + imgs
 
@@ -174,6 +174,7 @@ def worker():
         adm_scaler_negative = args.pop()
         adm_scaler_end = args.pop()
         adaptive_cfg = args.pop()
+        clip_skip = args.pop()
         sampler_name = args.pop()
         scheduler_name = args.pop()
         vae_name = args.pop()
@@ -237,10 +238,12 @@ def worker():
 
         steps = performance_selection.steps()
 
+        performance_loras = []
+
         if performance_selection == Performance.EXTREME_SPEED:
             print('Enter LCM mode.')
             progressbar(async_task, 1, 'Downloading LCM components ...')
-            loras += [(modules.config.downloading_sdxl_lcm_lora(), 1.0)]
+            performance_loras += [(modules.config.downloading_sdxl_lcm_lora(), 1.0)]
 
             if refiner_model_name != 'None':
                 print(f'Refiner disabled in LCM mode.')
@@ -259,7 +262,7 @@ def worker():
         elif performance_selection == Performance.LIGHTNING:
             print('Enter Lightning mode.')
             progressbar(async_task, 1, 'Downloading Lightning components ...')
-            loras += [(modules.config.downloading_sdxl_lightning_lora(), 1.0)]
+            performance_loras += [(modules.config.downloading_sdxl_lightning_lora(), 1.0)]
 
             if refiner_model_name != 'None':
                 print(f'Refiner disabled in Lightning mode.')
@@ -278,7 +281,7 @@ def worker():
         elif performance_selection == Performance.HYPER_SD:
             print('Enter Hyper-SD mode.')
             progressbar(async_task, 1, 'Downloading Hyper-SD components ...')
-            loras += [(modules.config.downloading_sdxl_hyper_sd_lora(), 0.8)]
+            performance_loras += [(modules.config.downloading_sdxl_hyper_sd_lora(), 0.8)]
 
             if refiner_model_name != 'None':
                 print(f'Refiner disabled in Hyper-SD mode.')
@@ -294,15 +297,8 @@ def worker():
             adm_scaler_negative = 1.0
             adm_scaler_end = 0.0
 
-        elif performance_selection == Performance.HYPER_SD8:
-            print('Enter Hyper-SD8 mode.')
-            progressbar(async_task, 1, 'Downloading Hyper-SD components ...')
-            loras += [(modules.config.downloading_sdxl_hyper_sd_cfg_lora(), 0.3)]
-
-            sampler_name = 'dpmpp_sde_gpu'
-            scheduler_name = 'normal'
-
         print(f'[Parameters] Adaptive CFG = {adaptive_cfg}')
+        print(f'[Parameters] CLIP Skip = {clip_skip}')
         print(f'[Parameters] Sharpness = {sharpness}')
         print(f'[Parameters] ControlNet Softness = {controlnet_softness}')
         print(f'[Parameters] ADM Scale = '
@@ -464,14 +460,16 @@ def worker():
             extra_positive_prompts = prompts[1:] if len(prompts) > 1 else []
             extra_negative_prompts = negative_prompts[1:] if len(negative_prompts) > 1 else []
 
-            progressbar(async_task, 3, 'Loading models ...')
+            progressbar(async_task, 2, 'Loading models ...')
-
-            loras = parse_lora_references_from_prompt(prompt, loras, modules.config.default_max_lora_number)
 
+            loras, prompt = parse_lora_references_from_prompt(prompt, loras, modules.config.default_max_lora_number)
+            loras += performance_loras
             pipeline.refresh_everything(refiner_model_name=refiner_model_name, base_model_name=base_model_name,
                                         loras=loras, base_model_additional_loras=base_model_additional_loras,
                                         use_synthetic_refiner=use_synthetic_refiner, vae_name=vae_name)
 
+            pipeline.set_clip_skip(clip_skip)
+
             progressbar(async_task, 3, 'Processing prompts ...')
             tasks = []
 
@@ -531,25 +529,25 @@ def worker():
 
             if use_expansion:
                 for i, t in enumerate(tasks):
-                    progressbar(async_task, 5, f'Preparing Fooocus text #{i + 1} ...')
+                    progressbar(async_task, 4, f'Preparing Fooocus text #{i + 1} ...')
                     expansion = pipeline.final_expansion(t['task_prompt'], t['task_seed'])
                     print(f'[Prompt Expansion] {expansion}')
                     t['expansion'] = expansion
                     t['positive'] = copy.deepcopy(t['positive']) + [expansion]  # Deep copy.
 
             for i, t in enumerate(tasks):
-                progressbar(async_task, 7, f'Encoding positive #{i + 1} ...')
+                progressbar(async_task, 5, f'Encoding positive #{i + 1} ...')
                 t['c'] = pipeline.clip_encode(texts=t['positive'], pool_top_k=t['positive_top_k'])
 
             for i, t in enumerate(tasks):
                 if abs(float(cfg_scale) - 1.0) < 1e-4:
                     t['uc'] = pipeline.clone_cond(t['c'])
                 else:
-                    progressbar(async_task, 10, f'Encoding negative #{i + 1} ...')
+                    progressbar(async_task, 6, f'Encoding negative #{i + 1} ...')
                     t['uc'] = pipeline.clip_encode(texts=t['negative'], pool_top_k=t['negative_top_k'])
 
         if len(goals) > 0:
-            progressbar(async_task, 13, 'Image processing ...')
+            progressbar(async_task, 7, 'Image processing ...')
 
         if 'vary' in goals:
             if 'subtle' in uov_method:
@@ -570,7 +568,7 @@ def worker():
             uov_input_image = set_image_shape_ceil(uov_input_image, shape_ceil)
 
             initial_pixels = core.numpy_to_pytorch(uov_input_image)
-            progressbar(async_task, 13, 'VAE encoding ...')
+            progressbar(async_task, 8, 'VAE encoding ...')
 
             candidate_vae, _ = pipeline.get_candidate_vae(
                 steps=steps,
@@ -587,7 +585,7 @@ def worker():
 
         if 'upscale' in goals:
             H, W, C = uov_input_image.shape
-            progressbar(async_task, 13, f'Upscaling image from {str((H, W))} ...')
+            progressbar(async_task, 9, f'Upscaling image from {str((H, W))} ...')
             uov_input_image = perform_upscale(uov_input_image)
             print(f'Image upscaled.')
 
@@ -623,7 +621,8 @@ def worker():
                 d = [('Upscale (Fast)', 'upscale_fast', '2x')]
                 if modules.config.default_black_out_nsfw or black_out_nsfw:
                     progressbar(async_task, 100, 'Checking for NSFW content ...')
-                    uov_input_image = censor_single(uov_input_image)
+                    uov_input_image = default_censor(uov_input_image)
+                progressbar(async_task, 100, 'Saving image to system ...')
                 uov_input_image_path = log(uov_input_image, d, output_format=output_format)
                 yield_result(async_task, uov_input_image_path, black_out_nsfw, False, do_not_show_finished_images=True)
                 return
@@ -635,7 +634,7 @@ def worker():
                 denoising_strength = overwrite_upscale_strength
 
             initial_pixels = core.numpy_to_pytorch(uov_input_image)
-            progressbar(async_task, 13, 'VAE encoding ...')
+            progressbar(async_task, 10, 'VAE encoding ...')
 
             candidate_vae, _ = pipeline.get_candidate_vae(
                 steps=steps,
@@ -693,7 +692,7 @@ def worker():
                              do_not_show_finished_images=True)
                 return
 
-            progressbar(async_task, 13, 'VAE Inpaint encoding ...')
+            progressbar(async_task, 11, 'VAE Inpaint encoding ...')
 
             inpaint_pixel_fill = core.numpy_to_pytorch(inpaint_worker.current_task.interested_fill)
             inpaint_pixel_image = core.numpy_to_pytorch(inpaint_worker.current_task.interested_image)
@@ -713,7 +712,7 @@ def worker():
 
             latent_swap = None
             if candidate_vae_swap is not None:
-                progressbar(async_task, 13, 'VAE SD15 encoding ...')
+                progressbar(async_task, 12, 'VAE SD15 encoding ...')
                 latent_swap = core.encode_vae(
                     vae=candidate_vae_swap,
                     pixels=inpaint_pixel_fill)['samples']
@@ -839,16 +838,17 @@ def worker():
                     zsnr=False)[0]
             print(f'Using {scheduler_name} scheduler.')
 
-        async_task.yields.append(['preview', (13, 'Moving model to GPU ...', None)])
+        async_task.yields.append(['preview', (flags.preparation_step_count, 'Moving model to GPU ...', None)])
 
         def callback(step, x0, x, total_steps, y):
             done_steps = current_task_id * steps + step
             async_task.yields.append(['preview', (
-                int(15.0 + 85.0 * float(done_steps) / float(all_steps)),
+                int(flags.preparation_step_count + (100 - flags.preparation_step_count) * float(done_steps) / float(all_steps)),
-                f'Step {step}/{total_steps} in the {current_task_id + 1}{ordinal_suffix(current_task_id + 1)} Sampling',
+                f'Sampling step {step + 1}/{total_steps}, image {current_task_id + 1}/{image_number} ...', y)])
-                y)])
 
         for current_task_id, task in enumerate(tasks):
+            current_progress = int(flags.preparation_step_count + (100 - flags.preparation_step_count) * float(current_task_id * steps) / float(all_steps))
+            progressbar(async_task, current_progress, f'Preparing task {current_task_id + 1}/{image_number} ...')
             execution_start_time = time.perf_counter()
 
             try:
@@ -891,12 +891,12 @@ def worker():
                     imgs = [inpaint_worker.current_task.post_process(x) for x in imgs]
 
                 img_paths = []
-
+                current_progress = int(flags.preparation_step_count + (100 - flags.preparation_step_count) * float((current_task_id + 1) * steps) / float(all_steps))
                 if modules.config.default_black_out_nsfw or black_out_nsfw:
-                    progressbar(async_task, int(15.0 + 85.0 * float((current_task_id + 1) * steps) / float(all_steps)),
+                    progressbar(async_task, current_progress, 'Checking for NSFW content ...')
-                                'Checking for NSFW content ...')
+                    imgs = default_censor(imgs)
-                    imgs = censor_batch(imgs)
 
+                progressbar(async_task, current_progress, f'Saving image {current_task_id + 1}/{image_number} to system ...')
                 for x in imgs:
                     d = [('Prompt', 'prompt', task['log_positive_prompt']),
                          ('Negative Prompt', 'negative_prompt', task['log_negative_prompt']),
@@ -928,6 +928,8 @@ def worker():
                         d.append(
                             ('CFG Mimicking from TSNR', 'adaptive_cfg', modules.patch.patch_settings[pid].adaptive_cfg))
 
+                    if clip_skip > 1:
+                        d.append(('CLIP Skip', 'clip_skip', clip_skip))
                     d.append(('Sampler', 'sampler', sampler_name))
                     d.append(('Scheduler', 'scheduler', scheduler_name))
                     d.append(('VAE', 'vae', vae_name))

modules/config.py

@@ -8,8 +8,7 @@ import modules.flags
 import modules.sdxl_styles
 
 from modules.model_loader import load_file_from_url
-from modules.util import makedirs_with_log
+from modules.extra_utils import makedirs_with_log, get_files_from_folder
-from modules.extra_utils import get_files_from_folder
 from modules.flags import OutputFormat, Performance, MetadataScheme
 
 
@@ -417,13 +416,7 @@ embeddings_downloads = get_config_item_or_set_default(
 )
 available_aspect_ratios = get_config_item_or_set_default(
     key='available_aspect_ratios',
-    default_value=[
+    default_value=modules.flags.sdxl_aspect_ratios,
-        '704*1408', '704*1344', '768*1344', '768*1280', '832*1216', '832*1152',
-        '896*1152', '896*1088', '960*1088', '960*1024', '1024*1024', '1024*960',
-        '1088*960', '1088*896', '1152*896', '1152*832', '1216*832', '1280*768',
-        '1344*768', '1344*704', '1408*704', '1472*704', '1536*640', '1600*640',
-        '1664*576', '1728*576'
-    ],
     validator=lambda x: isinstance(x, list) and all('*' in v for v in x) and len(x) > 1
 )
 default_aspect_ratio = get_config_item_or_set_default(
@@ -441,6 +434,11 @@ default_cfg_tsnr = get_config_item_or_set_default(
     default_value=7.0,
     validator=lambda x: isinstance(x, numbers.Number)
 )
+default_clip_skip = get_config_item_or_set_default(
+    key='default_clip_skip',
+    default_value=2,
+    validator=lambda x: isinstance(x, int) and 1 <= x <= modules.flags.clip_skip_max
+)
 default_overwrite_step = get_config_item_or_set_default(
     key='default_overwrite_step',
     default_value=-1,
@@ -494,6 +492,8 @@ possible_preset_keys = {
     "default_loras": "<processed>",
     "default_cfg_scale": "guidance_scale",
     "default_sample_sharpness": "sharpness",
+    "default_cfg_tsnr": "adaptive_cfg",
+    "default_clip_skip": "clip_skip",
     "default_sampler": "sampler",
     "default_scheduler": "scheduler",
     "default_overwrite_step": "steps",
@@ -527,7 +527,7 @@ def add_ratio(x):
 
 
 default_aspect_ratio = add_ratio(default_aspect_ratio)
-available_aspect_ratios = [add_ratio(x) for x in available_aspect_ratios]
+available_aspect_ratios_labels = [add_ratio(x) for x in available_aspect_ratios]
 
 
 # Only write config in the first launch.
@@ -548,6 +548,7 @@ with open(config_example_path, "w", encoding="utf-8") as json_file:
 
 model_filenames = []
 lora_filenames = []
+lora_filenames_no_special = []
 vae_filenames = []
 wildcard_filenames = []
 
@@ -557,6 +558,16 @@ sdxl_hyper_sd_lora = 'sdxl_hyper_sd_4step_lora.safetensors'
 loras_metadata_remove = [sdxl_lcm_lora, sdxl_lightning_lora, sdxl_hyper_sd_lora]
 
 
+def remove_special_loras(lora_filenames):
+    global loras_metadata_remove
+
+    loras_no_special = lora_filenames.copy()
+    for lora_to_remove in loras_metadata_remove:
+        if lora_to_remove in loras_no_special:
+            loras_no_special.remove(lora_to_remove)
+    return loras_no_special
+
+
 def get_model_filenames(folder_paths, extensions=None, name_filter=None):
     if extensions is None:
         extensions = ['.pth', '.ckpt', '.bin', '.safetensors', '.fooocus.patch']
@@ -571,9 +582,10 @@ def get_model_filenames(folder_paths, extensions=None, name_filter=None):
 
 
 def update_files():
-    global model_filenames, lora_filenames, vae_filenames, wildcard_filenames, available_presets
+    global model_filenames, lora_filenames, lora_filenames_no_special, vae_filenames, wildcard_filenames, available_presets
     model_filenames = get_model_filenames(paths_checkpoints)
     lora_filenames = get_model_filenames(paths_loras)
+    lora_filenames_no_special = remove_special_loras(lora_filenames)
     vae_filenames = get_model_filenames(path_vae)
     wildcard_filenames = get_files_from_folder(path_wildcards, ['.txt'])
     available_presets = get_presets()

modules/default_pipeline.py

@@ -201,6 +201,17 @@ def clip_encode(texts, pool_top_k=1):
     return [[torch.cat(cond_list, dim=1), {"pooled_output": pooled_acc}]]
 
 
+@torch.no_grad()
+@torch.inference_mode()
+def set_clip_skip(clip_skip: int):
+    global final_clip
+
+    if final_clip is None:
+        return
+
+    final_clip.clip_layer(-abs(clip_skip))
+    return
+
 @torch.no_grad()
 @torch.inference_mode()
 def clear_all_caches():

modules/extra_utils.py

@@ -1,5 +1,11 @@
 import os
 
+def makedirs_with_log(path):
+    try:
+        os.makedirs(path, exist_ok=True)
+    except OSError as error:
+        print(f'Directory {path} could not be created, reason: {error}')
+
 
 def get_files_from_folder(folder_path, extensions=None, name_filter=None):
     if not os.path.isdir(folder_path):

modules/flags.py

@@ -54,6 +54,8 @@ SAMPLER_NAMES = KSAMPLER_NAMES + list(SAMPLER_EXTRA.keys())
 sampler_list = SAMPLER_NAMES
 scheduler_list = SCHEDULER_NAMES
 
+clip_skip_max = 12
+
 default_vae = 'Default (model)'
 
 refiner_swap_method = 'joint'
@@ -81,6 +83,13 @@ inpaint_options = [inpaint_option_default, inpaint_option_detail, inpaint_option
 desc_type_photo = 'Photograph'
 desc_type_anime = 'Art/Anime'
 
+sdxl_aspect_ratios = [
+    '704*1408', '704*1344', '768*1344', '768*1280', '832*1216', '832*1152',
+    '896*1152', '896*1088', '960*1088', '960*1024', '1024*1024', '1024*960',
+    '1088*960', '1088*896', '1152*896', '1152*832', '1216*832', '1280*768',
+    '1344*768', '1344*704', '1408*704', '1472*704', '1536*640', '1600*640',
+    '1664*576', '1728*576'
+]
 
 class MetadataScheme(Enum):
     FOOOCUS = 'fooocus'
@@ -93,6 +102,7 @@ metadata_scheme = [
 ]
 
 controlnet_image_count = 4
+preparation_step_count = 13
 
 
 class OutputFormat(Enum):

modules/meta_parser.py

@@ -34,16 +34,17 @@ def load_parameter_button_click(raw_metadata: dict | str, is_generating: bool):
     get_list('styles', 'Styles', loaded_parameter_dict, results)
     get_str('performance', 'Performance', loaded_parameter_dict, results)
     get_steps('steps', 'Steps', loaded_parameter_dict, results)
-    get_float('overwrite_switch', 'Overwrite Switch', loaded_parameter_dict, results)
+    get_number('overwrite_switch', 'Overwrite Switch', loaded_parameter_dict, results)
     get_resolution('resolution', 'Resolution', loaded_parameter_dict, results)
-    get_float('guidance_scale', 'Guidance Scale', loaded_parameter_dict, results)
+    get_number('guidance_scale', 'Guidance Scale', loaded_parameter_dict, results)
-    get_float('sharpness', 'Sharpness', loaded_parameter_dict, results)
+    get_number('sharpness', 'Sharpness', loaded_parameter_dict, results)
     get_adm_guidance('adm_guidance', 'ADM Guidance', loaded_parameter_dict, results)
     get_str('refiner_swap_method', 'Refiner Swap Method', loaded_parameter_dict, results)
-    get_float('adaptive_cfg', 'CFG Mimicking from TSNR', loaded_parameter_dict, results)
+    get_number('adaptive_cfg', 'CFG Mimicking from TSNR', loaded_parameter_dict, results)
+    get_number('clip_skip', 'CLIP Skip', loaded_parameter_dict, results, cast_type=int)
     get_str('base_model', 'Base Model', loaded_parameter_dict, results)
     get_str('refiner_model', 'Refiner Model', loaded_parameter_dict, results)
-    get_float('refiner_switch', 'Refiner Switch', loaded_parameter_dict, results)
+    get_number('refiner_switch', 'Refiner Switch', loaded_parameter_dict, results)
     get_str('sampler', 'Sampler', loaded_parameter_dict, results)
     get_str('scheduler', 'Scheduler', loaded_parameter_dict, results)
     get_str('vae', 'VAE', loaded_parameter_dict, results)
@@ -83,11 +84,11 @@ def get_list(key: str, fallback: str | None, source_dict: dict, results: list, d
         results.append(gr.update())
 
 
-def get_float(key: str, fallback: str | None, source_dict: dict, results: list, default=None):
+def get_number(key: str, fallback: str | None, source_dict: dict, results: list, default=None, cast_type=float):
     try:
         h = source_dict.get(key, source_dict.get(fallback, default))
         assert h is not None
-        h = float(h)
+        h = cast_type(h)
         results.append(h)
     except:
         results.append(gr.update())
@@ -124,7 +125,7 @@ def get_resolution(key: str, fallback: str | None, source_dict: dict, results: l
         h = source_dict.get(key, source_dict.get(fallback, default))
         width, height = eval(h)
         formatted = modules.config.add_ratio(f'{width}*{height}')
-        if formatted in modules.config.available_aspect_ratios:
+        if formatted in modules.config.available_aspect_ratios_labels:
             results.append(formatted)
             results.append(-1)
             results.append(-1)
@@ -205,7 +206,6 @@ def get_lora(key: str, fallback: str | None, source_dict: dict, results: list):
 def get_sha256(filepath):
     global hash_cache
     if filepath not in hash_cache:
-        # is_safetensors = os.path.splitext(filepath)[1].lower() == '.safetensors'
         hash_cache[filepath] = sha256(filepath)
 
     return hash_cache[filepath]
@@ -293,12 +293,6 @@ class MetadataParser(ABC):
                 self.loras.append((Path(lora_name).stem, lora_weight, lora_hash))
         self.vae_name = Path(vae_name).stem
 
-    @staticmethod
-    def remove_special_loras(lora_filenames):
-        for lora_to_remove in modules.config.loras_metadata_remove:
-            if lora_to_remove in lora_filenames:
-                lora_filenames.remove(lora_to_remove)
-
 
 class A1111MetadataParser(MetadataParser):
     def get_scheme(self) -> MetadataScheme:
@@ -321,6 +315,7 @@ class A1111MetadataParser(MetadataParser):
         'adm_guidance': 'ADM Guidance',
         'refiner_swap_method': 'Refiner Swap Method',
         'adaptive_cfg': 'Adaptive CFG',
+        'clip_skip': 'Clip skip',
         'overwrite_switch': 'Overwrite Switch',
         'freeu': 'FreeU',
         'base_model': 'Model',
@@ -415,13 +410,11 @@ class A1111MetadataParser(MetadataParser):
             lora_data = data['lora_hashes']
 
         if lora_data != '':
-            lora_filenames = modules.config.lora_filenames.copy()
-            self.remove_special_loras(lora_filenames)
             for li, lora in enumerate(lora_data.split(', ')):
                 lora_split = lora.split(': ')
                 lora_name = lora_split[0]
                 lora_weight = lora_split[2] if len(lora_split) == 3 else lora_split[1]
-                for filename in lora_filenames:
+                for filename in modules.config.lora_filenames_no_special:
                     path = Path(filename)
                     if lora_name == path.stem:
                         data[f'lora_combined_{li + 1}'] = f'{filename} : {lora_weight}'
@@ -467,7 +460,7 @@ class A1111MetadataParser(MetadataParser):
                 self.fooocus_to_a1111['refiner_model_hash']: self.refiner_model_hash
             }
 
-        for key in ['adaptive_cfg', 'overwrite_switch', 'refiner_swap_method', 'freeu']:
+        for key in ['adaptive_cfg', 'clip_skip', 'overwrite_switch', 'refiner_swap_method', 'freeu']:
             if key in data:
                 generation_params[self.fooocus_to_a1111[key]] = data[key]
 
@@ -510,19 +503,15 @@ class FooocusMetadataParser(MetadataParser):
         return MetadataScheme.FOOOCUS
 
     def parse_json(self, metadata: dict) -> dict:
-        model_filenames = modules.config.model_filenames.copy()
-        lora_filenames = modules.config.lora_filenames.copy()
-        vae_filenames = modules.config.vae_filenames.copy()
-        self.remove_special_loras(lora_filenames)
         for key, value in metadata.items():
             if value in ['', 'None']:
                 continue
             if key in ['base_model', 'refiner_model']:
-                metadata[key] = self.replace_value_with_filename(key, value, model_filenames)
+                metadata[key] = self.replace_value_with_filename(key, value, modules.config.model_filenames)
             elif key.startswith('lora_combined_'):
-                metadata[key] = self.replace_value_with_filename(key, value, lora_filenames)
+                metadata[key] = self.replace_value_with_filename(key, value, modules.config.lora_filenames_no_special)
             elif key == 'vae':
-                metadata[key] = self.replace_value_with_filename(key, value, vae_filenames)
+                metadata[key] = self.replace_value_with_filename(key, value, modules.config.vae_filenames)
             else:
                 continue
 

modules/util.py

@@ -1,3 +1,5 @@
+from pathlib import Path
+
 import numpy as np
 import datetime
 import random
@@ -12,15 +14,15 @@ import hashlib
 
 from PIL import Image
 
+import modules.config
 import modules.sdxl_styles
 
 LANCZOS = (Image.Resampling.LANCZOS if hasattr(Image, 'Resampling') else Image.LANCZOS)
 
-
 # Regexp compiled once. Matches entries with the following pattern:
 # <lora:some_lora:1>
 # <lora:aNotherLora:-1.6>
-LORAS_PROMPT_PATTERN = re.compile(r".* <lora : ([^:]+) : ([+-]? (?: (?:\d+ (?:\.\d*)?) | (?:\.\d+)))> .*", re.X)
+LORAS_PROMPT_PATTERN = re.compile(r"(<lora:([^:]+):([+-]?(?:\d+(?:\.\d*)?|\.\d+))>)", re.X)
 
 HASH_SHA256_LENGTH = 10
 
@@ -360,6 +362,14 @@ def is_json(data: str) -> bool:
     return True
 
 
+def get_filname_by_stem(lora_name, filenames: List[str]) -> str | None:
+    for filename in filenames:
+        path = Path(filename)
+        if lora_name == path.stem:
+            return filename
+    return None
+
+
 def get_file_from_folder_list(name, folders):
     if not isinstance(folders, list):
         folders = [folders]
@@ -371,7 +381,6 @@ def get_file_from_folder_list(name, folders):
 
     return os.path.abspath(os.path.realpath(os.path.join(folders[0], name)))
 
-
 def ordinal_suffix(number: int) -> str:
     return 'th' if 10 <= number % 100 <= 20 else {1: 'st', 2: 'nd', 3: 'rd'}.get(number % 10, 'th')
 
@@ -383,24 +392,64 @@ def makedirs_with_log(path):
         print(f'Directory {path} could not be created, reason: {error}')
 
 
-def get_enabled_loras(loras: list) -> list:
+def get_enabled_loras(loras: list, remove_none=True) -> list:
-    return [(lora[1], lora[2]) for lora in loras if lora[0]]
+    return [(lora[1], lora[2]) for lora in loras if lora[0] and (lora[1] != 'None' if remove_none else True)]
 
 
-def parse_lora_references_from_prompt(prompt: str, loras: List[Tuple[AnyStr, float]], loras_limit: int = 5) -> List[Tuple[AnyStr, float]]:
+def parse_lora_references_from_prompt(prompt: str, loras: List[Tuple[AnyStr, float]], loras_limit: int = 5,
+                                      skip_file_check=False, prompt_cleanup=True, deduplicate_loras=True) -> tuple[List[Tuple[AnyStr, float]], str]:
+    found_loras = []
+    prompt_without_loras = ''
+    cleaned_prompt = ''
+    for token in prompt.split(','):
+        matches = LORAS_PROMPT_PATTERN.findall(token)
+
+        if len(matches) == 0:
+            prompt_without_loras += token + ', '
+            continue
+        for match in matches:
+            lora_name = match[1] + '.safetensors'
+            if not skip_file_check:
+                lora_name = get_filname_by_stem(match[1], modules.config.lora_filenames_no_special)
+            if lora_name is not None:
+                found_loras.append((lora_name, float(match[2])))
+            token = token.replace(match[0], '')
+        prompt_without_loras += token + ', '
+
+    if prompt_without_loras != '':
+        cleaned_prompt = prompt_without_loras[:-2]
+
+    if prompt_cleanup:
+        cleaned_prompt = cleanup_prompt(prompt_without_loras)
+
     new_loras = []
+    lora_names = [lora[0] for lora in loras]
+    for found_lora in found_loras:
+        if deduplicate_loras and (found_lora[0] in lora_names or found_lora in new_loras):
+            continue
+        new_loras.append(found_lora)
+
+    if len(new_loras) == 0:
+        return loras, cleaned_prompt
+
     updated_loras = []
-    for token in prompt.split(","):
-        m = LORAS_PROMPT_PATTERN.match(token)
-
-        if m:
-            new_loras.append((f"{m.group(1)}.safetensors", float(m.group(2))))
-
     for lora in loras + new_loras:
         if lora[0] != "None":
             updated_loras.append(lora)
 
-    return updated_loras[:loras_limit]
+    return updated_loras[:loras_limit], cleaned_prompt
+
+
+def cleanup_prompt(prompt):
+    prompt = re.sub(' +', ' ', prompt)
+    prompt = re.sub(',+', ',', prompt)
+    cleaned_prompt = ''
+    for token in prompt.split(','):
+        token = token.strip()
+        if token == '':
+            continue
+        cleaned_prompt += token + ', '
+    return cleaned_prompt[:-2]
 
 
 def apply_wildcards(wildcard_text, rng, i, read_wildcards_in_order) -> str:
@@ -428,3 +477,26 @@ def apply_wildcards(wildcard_text, rng, i, read_wildcards_in_order) -> str:
 
     print(f'[Wildcards] BFS stack overflow. Current text: {wildcard_text}')
     return wildcard_text
+
+
+def get_image_size_info(image: np.ndarray, aspect_ratios: list) -> str:
+    try:
+        image = Image.fromarray(np.uint8(image))
+        width, height = image.size
+        ratio = round(width / height, 2)
+        gcd = math.gcd(width, height)
+        lcm_ratio = f'{width // gcd}:{height // gcd}'
+        size_info = f'Image Size: {width} x {height}, Ratio: {ratio}, {lcm_ratio}'
+
+        closest_ratio = min(aspect_ratios, key=lambda x: abs(ratio - float(x.split('*')[0]) / float(x.split('*')[1])))
+        recommended_width, recommended_height = map(int, closest_ratio.split('*'))
+        recommended_ratio = round(recommended_width / recommended_height, 2)
+        recommended_gcd = math.gcd(recommended_width, recommended_height)
+        recommended_lcm_ratio = f'{recommended_width // recommended_gcd}:{recommended_height // recommended_gcd}'
+
+        size_info = f'{width} x {height}, {ratio}, {lcm_ratio}'
+        size_info += f'\n{recommended_width} x {recommended_height}, {recommended_ratio}, {recommended_lcm_ratio}'
+
+        return size_info
+    except Exception as e:
+        return f'Error reading image: {e}'

requirements_docker.txt

@@ -1,5 +1,2 @@
-torch==2.0.1
+torch==2.1.0
-torchvision==0.15.2
+torchvision==0.16.0
-torchaudio==2.0.2
-torchtext==0.15.2
-torchdata==0.6.1

tests/test_utils.py

@@ -7,13 +7,17 @@ class TestUtils(unittest.TestCase):
     def test_can_parse_tokens_with_lora(self):
         test_cases = [
             {
-                "input": ("some prompt, very cool, <lora:hey-lora:0.4>,  cool   <lora:you-lora:0.2>", [], 5),
+                "input": ("some prompt, very cool, <lora:hey-lora:0.4>,  cool   <lora:you-lora:0.2>", [], 5, True),
-                "output": [("hey-lora.safetensors", 0.4), ("you-lora.safetensors", 0.2)],
+                "output": (
+                    [('hey-lora.safetensors', 0.4), ('you-lora.safetensors', 0.2)], 'some prompt, very cool, cool'),
             },
             # Test can not exceed limit
             {
-                "input": ("some prompt, very cool, <lora:hey-lora:0.4>,  cool   <lora:you-lora:0.2>", [], 1),
+                "input": ("some prompt, very cool, <lora:hey-lora:0.4>,  cool   <lora:you-lora:0.2>", [], 1, True),
-                "output": [("hey-lora.safetensors", 0.4)],
+                "output": (
+                    [('hey-lora.safetensors', 0.4)],
+                    'some prompt, very cool, cool'
+                ),
             },
             # test Loras from UI take precedence over prompt
             {
@@ -21,28 +25,57 @@ class TestUtils(unittest.TestCase):
                     "some prompt, very cool, <lora:l1:0.4>, <lora:l2:-0.2>, <lora:l3:0.3>, <lora:l4:0.5>, <lora:l6:0.24>, <lora:l7:0.1>",
                     [("hey-lora.safetensors", 0.4)],
                     5,
+                    True
                 ),
-                "output": [
+                "output": (
-                    ("hey-lora.safetensors", 0.4),
+                    [
-                    ("l1.safetensors", 0.4),
+                        ('hey-lora.safetensors', 0.4),
-                    ("l2.safetensors", -0.2),
+                        ('l1.safetensors', 0.4),
-                    ("l3.safetensors", 0.3),
+                        ('l2.safetensors', -0.2),
-                    ("l4.safetensors", 0.5),
+                        ('l3.safetensors', 0.3),
-                ],
+                        ('l4.safetensors', 0.5)
+                    ],
+                    'some prompt, very cool'
+                )
             },
-            # Test lora specification not separated by comma are ignored, only latest specified is used
+            # test correct matching even if there is no space separating loras in the same token
             {
-                "input": ("some prompt, very cool, <lora:hey-lora:0.4><lora:you-lora:0.2>", [], 3),
+                "input": ("some prompt, very cool, <lora:hey-lora:0.4><lora:you-lora:0.2>", [], 3, True),
-                "output": [("you-lora.safetensors", 0.2)],
+                "output": (
+                    [
+                        ('hey-lora.safetensors', 0.4),
+                        ('you-lora.safetensors', 0.2)
+                    ],
+                    'some prompt, very cool'
+                ),
+            },
+            # test deduplication, also selected loras are never overridden with loras in prompt
+            {
+                "input": (
+                    "some prompt, very cool, <lora:hey-lora:0.4><lora:hey-lora:0.4><lora:you-lora:0.2>",
+                    [('you-lora.safetensors', 0.3)],
+                    3,
+                    True
+                ),
+                "output": (
+                    [
+                        ('you-lora.safetensors', 0.3),
+                        ('hey-lora.safetensors', 0.4)
+                    ],
+                    'some prompt, very cool'
+                ),
             },
             {
-                "input": ("<lora:foo:1..2>, <lora:bar:.>, <lora:baz:+> and <lora:quux:>", [], 6),
+                "input": ("<lora:foo:1..2>, <lora:bar:.>, <test:1.0>, <lora:baz:+> and <lora:quux:>", [], 6, True),
-                "output": []
+                "output": (
+                    [],
+                    '<lora:foo:1..2>, <lora:bar:.>, <test:1.0>, <lora:baz:+> and <lora:quux:>'
+                )
             }
         ]
 
         for test in test_cases:
-            prompt, loras, loras_limit = test["input"]
+            prompt, loras, loras_limit, skip_file_check = test["input"]
             expected = test["output"]
-            actual = util.parse_lora_references_from_prompt(prompt, loras, loras_limit)
+            actual = util.parse_lora_references_from_prompt(prompt, loras, loras_limit=loras_limit, skip_file_check=skip_file_check)
             self.assertEqual(expected, actual)

update_log.md

@@ -1,3 +1,28 @@
+# [2.4.1](https://github.com/lllyasviel/Fooocus/releases/tag/v2.4.1)
+
+* Fix some small bugs (e.g. adjust clip skip default value from 1 to 2, add type check to aspect ratios js update function)
+* Add automated docker build on push to main, tagged with `edge`. See [available docker images](https://github.com/lllyasviel/Fooocus/pkgs/container/fooocus).
+
+# [2.4.0](https://github.com/lllyasviel/Fooocus/releases/tag/v2.4.0)
+
+* Change settings tab elements to be more compact
+* Add clip skip slider
+* Add select for custom VAE
+* Add new style "Random Style"
+* Update default anime model to animaPencilXL_v310
+* Add button to reconnect the UI after Fooocus crashed without having to configure everything again (no page reload required)
+* Add performance "hyper-sd" (based on [Hyper-SDXL 4 step LoRA](https://huggingface.co/ByteDance/Hyper-SD/blob/main/Hyper-SDXL-4steps-lora.safetensors))
+* Add [AlignYourSteps](https://research.nvidia.com/labs/toronto-ai/AlignYourSteps/) scheduler by Nvidia, see 
+* Add [TCD](https://github.com/jabir-zheng/TCD) sampler and scheduler (based on sgm_uniform)
+* Add NSFW image censoring (disables intermediate image preview while generating). Set config value `default_black_out_nsfw` to True to always enable.
+* Add argument `--enable-describe-uov-image` to automatically describe uploaded images for upscaling
+* Add inline lora prompt references with subfolder support, example prompt: `colorful bird <lora:toucan:1.2>`
+* Add size and aspect ratio recommendation on image describe
+* Add inpaint brush color picker, helpful when image and mask brush have the same color
+* Add automated Docker image build using Github Actions on each release.
+* Add full raw prompts to history logs
+* Change code ownership from @lllyasviel to @mashb1t for automated issue / MR notification
+
 # [2.3.1](https://github.com/lllyasviel/Fooocus/releases/tag/2.3.1)
 
 * Remove positive prompt from anime prefix to not reset prompt after switching presets

webui.py

@@ -152,7 +152,7 @@ with shared.gradio_root:
                     with gr.TabItem(label='Upscale or Variation') as uov_tab:
                         with gr.Row():
                             with gr.Column():
-                                uov_input_image = grh.Image(label='Drag above image to here', source='upload', type='numpy')
+                                uov_input_image = grh.Image(label='Image', source='upload', type='numpy', show_label=False)
                             with gr.Column():
                                 uov_method = gr.Radio(label='Upscale or Variation:', choices=flags.uov_list, value=flags.disabled)
                                 gr.HTML('<a href="https://github.com/lllyasviel/Fooocus/discussions/390" target="_blank">\U0001F4D4 Document</a>')
@@ -201,7 +201,7 @@ with shared.gradio_root:
                                            queue=False, show_progress=False)
                     with gr.TabItem(label='Inpaint or Outpaint') as inpaint_tab:
                         with gr.Row():
-                            inpaint_input_image = grh.Image(label='Drag inpaint or outpaint image to here', source='upload', type='numpy', tool='sketch', height=500, brush_color="#FFFFFF", elem_id='inpaint_canvas')
+                            inpaint_input_image = grh.Image(label='Image', source='upload', type='numpy', tool='sketch', height=500, brush_color="#FFFFFF", elem_id='inpaint_canvas', show_label=False)
                             inpaint_mask_image = grh.Image(label='Mask Upload', source='upload', type='numpy', height=500, visible=False)
 
                         with gr.Row():
@@ -214,17 +214,26 @@ with shared.gradio_root:
                     with gr.TabItem(label='Describe') as desc_tab:
                         with gr.Row():
                             with gr.Column():
-                                desc_input_image = grh.Image(label='Drag any image to here', source='upload', type='numpy')
+                                desc_input_image = grh.Image(label='Image', source='upload', type='numpy', show_label=False)
                             with gr.Column():
                                 desc_method = gr.Radio(
                                     label='Content Type',
                                     choices=[flags.desc_type_photo, flags.desc_type_anime],
                                     value=flags.desc_type_photo)
                                 desc_btn = gr.Button(value='Describe this Image into Prompt')
+                                desc_image_size = gr.Textbox(label='Image Size and Recommended Size', elem_id='desc_image_size', visible=False)
                                 gr.HTML('<a href="https://github.com/lllyasviel/Fooocus/discussions/1363" target="_blank">\U0001F4D4 Document</a>')
-                    with gr.TabItem(label='Metadata') as load_tab:
+
+                                def trigger_show_image_properties(image):
+                                    value = modules.util.get_image_size_info(image, modules.flags.sdxl_aspect_ratios)
+                                    return gr.update(value=value, visible=True)
+
+                                desc_input_image.upload(trigger_show_image_properties, inputs=desc_input_image,
+                                                        outputs=desc_image_size, show_progress=False, queue=False)
+
+                    with gr.TabItem(label='Metadata') as metadata_tab:
                         with gr.Column():
-                            metadata_input_image = grh.Image(label='Drag any image generated by Fooocus here', source='upload', type='filepath')
+                            metadata_input_image = grh.Image(label='For images created by Fooocus', source='upload', type='filepath')
                             metadata_json = gr.JSON(label='Metadata')
                             metadata_import_button = gr.Button(value='Apply Metadata')
 
@@ -255,25 +264,34 @@ with shared.gradio_root:
             inpaint_tab.select(lambda: 'inpaint', outputs=current_tab, queue=False, _js=down_js, show_progress=False)
             ip_tab.select(lambda: 'ip', outputs=current_tab, queue=False, _js=down_js, show_progress=False)
             desc_tab.select(lambda: 'desc', outputs=current_tab, queue=False, _js=down_js, show_progress=False)
+            metadata_tab.select(lambda: 'metadata', outputs=current_tab, queue=False, _js=down_js, show_progress=False)
 
         with gr.Column(scale=1, visible=modules.config.default_advanced_checkbox) as advanced_column:
             with gr.Tab(label='Setting'):
                 if not args_manager.args.disable_preset_selection:
-                    preset_selection = gr.Radio(label='Preset',
+                    preset_selection = gr.Dropdown(label='Preset',
-                                                choices=modules.config.available_presets,
+                                                   choices=modules.config.available_presets,
-                                                value=args_manager.args.preset if args_manager.args.preset else "initial",
+                                                   value=args_manager.args.preset if args_manager.args.preset else "initial",
-                                                interactive=True)
+                                                   interactive=True)
                 performance_selection = gr.Radio(label='Performance',
                                                  choices=flags.Performance.list(),
-                                                 value=modules.config.default_performance)
+                                                 value=modules.config.default_performance,
-                aspect_ratios_selection = gr.Radio(label='Aspect Ratios', choices=modules.config.available_aspect_ratios,
+                                                 elem_classes=['performance_selection'])
-                                                   value=modules.config.default_aspect_ratio, info='width × height',
+                with gr.Accordion(label='Aspect Ratios', open=False, elem_id='aspect_ratios_accordion') as aspect_ratios_accordion:
-                                                   elem_classes='aspect_ratios')
+                    aspect_ratios_selection = gr.Radio(label='Aspect Ratios', show_label=False,
+                                                       choices=modules.config.available_aspect_ratios_labels,
+                                                       value=modules.config.default_aspect_ratio,
+                                                       info='width × height',
+                                                       elem_classes='aspect_ratios')
+
+                    aspect_ratios_selection.change(lambda x: None, inputs=aspect_ratios_selection, queue=False, show_progress=False, _js='(x)=>{refresh_aspect_ratios_label(x);}')
+                    shared.gradio_root.load(lambda x: None, inputs=aspect_ratios_selection, queue=False, show_progress=False, _js='(x)=>{refresh_aspect_ratios_label(x);}')
+
                 image_number = gr.Slider(label='Image Number', minimum=1, maximum=modules.config.default_max_image_number, step=1, value=modules.config.default_image_number)
 
                 output_format = gr.Radio(label='Output Format',
-                                            choices=flags.OutputFormat.list(),
+                                         choices=flags.OutputFormat.list(),
-                                            value=modules.config.default_output_format)
+                                         value=modules.config.default_output_format)
 
                 negative_prompt = gr.Textbox(label='Negative Prompt', show_label=True, placeholder="Type prompt here.",
                                              info='Describing what you do not want to see.', lines=2,
@@ -403,6 +421,9 @@ with shared.gradio_root:
                                                  value=modules.config.default_cfg_tsnr,
                                                  info='Enabling Fooocus\'s implementation of CFG mimicking for TSNR '
                                                       '(effective when real CFG > mimicked CFG).')
+                        clip_skip = gr.Slider(label='CLIP Skip', minimum=1, maximum=flags.clip_skip_max, step=1,
+                                                 value=modules.config.default_clip_skip,
+                                                 info='Bypass CLIP layers to avoid overfitting (use 1 to not skip any layers, 2 is recommended).')
                         sampler_name = gr.Dropdown(label='Sampler', choices=flags.sampler_list,
                                                    value=modules.config.default_sampler)
                         scheduler_name = gr.Dropdown(label='Scheduler', choices=flags.scheduler_list,
@@ -515,13 +536,20 @@ with shared.gradio_root:
                         inpaint_mask_upload_checkbox = gr.Checkbox(label='Enable Mask Upload', value=False)
                         invert_mask_checkbox = gr.Checkbox(label='Invert Mask', value=False)
 
+                        inpaint_mask_color = gr.ColorPicker(label='Inpaint brush color', value='#FFFFFF', elem_id='inpaint_brush_color')
+
                         inpaint_ctrls = [debugging_inpaint_preprocessor, inpaint_disable_initial_latent, inpaint_engine,
                                          inpaint_strength, inpaint_respective_field,
                                          inpaint_mask_upload_checkbox, invert_mask_checkbox, inpaint_erode_or_dilate]
 
                         inpaint_mask_upload_checkbox.change(lambda x: gr.update(visible=x),
-                                                           inputs=inpaint_mask_upload_checkbox,
+                                                            inputs=inpaint_mask_upload_checkbox,
-                                                           outputs=inpaint_mask_image, queue=False, show_progress=False)
+                                                            outputs=inpaint_mask_image, queue=False,
+                                                            show_progress=False)
+
+                        inpaint_mask_color.change(lambda x: gr.update(brush_color=x), inputs=inpaint_mask_color,
+                                                  outputs=inpaint_input_image,
+                                                  queue=False, show_progress=False)
 
                     with gr.Tab(label='FreeU'):
                         freeu_enabled = gr.Checkbox(label='Enabled', value=False)
@@ -560,9 +588,9 @@ with shared.gradio_root:
         load_data_outputs = [advanced_checkbox, image_number, prompt, negative_prompt, style_selections,
                              performance_selection, overwrite_step, overwrite_switch, aspect_ratios_selection,
                              overwrite_width, overwrite_height, guidance_scale, sharpness, adm_scaler_positive,
-                             adm_scaler_negative, adm_scaler_end, refiner_swap_method, adaptive_cfg, base_model,
+                             adm_scaler_negative, adm_scaler_end, refiner_swap_method, adaptive_cfg, clip_skip,
-                             refiner_model, refiner_switch, sampler_name, scheduler_name, vae_name, seed_random,
+                             base_model, refiner_model, refiner_switch, sampler_name, scheduler_name, vae_name,
-                             image_seed, generate_button, load_parameter_button] + freeu_ctrls + lora_ctrls
+                             seed_random, image_seed, generate_button, load_parameter_button] + freeu_ctrls + lora_ctrls
 
         if not args_manager.args.disable_preset_selection:
             def preset_selection_change(preset, is_generating):
@@ -584,7 +612,7 @@ with shared.gradio_root:
                 return modules.meta_parser.load_parameter_button_click(json.dumps(preset_prepared), is_generating)
 
             preset_selection.change(preset_selection_change, inputs=[preset_selection, state_is_generating], outputs=load_data_outputs, queue=False, show_progress=True) \
-                .then(fn=style_sorter.sort_styles, inputs=style_selections, outputs=style_selections, queue=False, show_progress=False) \
+                .then(fn=style_sorter.sort_styles, inputs=style_selections, outputs=style_selections, queue=False, show_progress=False)
 
         performance_selection.change(lambda x: [gr.update(interactive=not flags.Performance.has_restricted_features(x))] * 11 +
                                                [gr.update(visible=not flags.Performance.has_restricted_features(x))] * 1 +
@@ -647,7 +675,7 @@ with shared.gradio_root:
         ctrls += [uov_method, uov_input_image]
         ctrls += [outpaint_selections, inpaint_input_image, inpaint_additional_prompt, inpaint_mask_image]
         ctrls += [disable_preview, disable_intermediate_results, disable_seed_increment, black_out_nsfw]
-        ctrls += [adm_scaler_positive, adm_scaler_negative, adm_scaler_end, adaptive_cfg]
+        ctrls += [adm_scaler_positive, adm_scaler_negative, adm_scaler_end, adaptive_cfg, clip_skip]
         ctrls += [sampler_name, scheduler_name, vae_name]
         ctrls += [overwrite_step, overwrite_switch, overwrite_width, overwrite_height, overwrite_vary_strength]
         ctrls += [overwrite_upscale_strength, mixing_image_prompt_and_vary_upscale, mixing_image_prompt_and_inpaint]

wildcards/.gitignore

@@ -0,0 +1,8 @@
+*.txt
+!animal.txt
+!artist.txt
+!color.txt
+!color_flower.txt
+!extended-color.txt
+!flower.txt
+!nationality.txt