Auto Extensions (#176)

Closes #148 
Closes #172

.devscripts/chmod.sh

@@ -2,4 +2,4 @@
 
 set -Eeuo pipefail
 
-find . -name "*.sh" -exec git update-index --chmod=+x {} \;
+find services -name "*.sh" -exec git update-index --chmod=+x {} \;

.github/pull_request_template.md

@@ -1,3 +1,11 @@
+<!--
+Have you created an issue before opening a merge request???
+https://github.com/AbdBarho/stable-diffusion-webui-docker#contributing
+Please create one so we can discuss it, I don't want your effort to go to waste.
+-->
+
+Closes issue #
+
 ### Update versions
 
 - auto: https://github.com/AUTOMATIC1111/stable-diffusion-webui/commit/

.github/workflows/docker.yml

@@ -1,6 +1,12 @@
 name: Build Images
 
-on: [push]
+on:
+  push:
+    branches: master
+  pull_request:
+      paths:
+        - docker-compose.yml
+        - services
 
 jobs:
   build:

LICENSE

@@ -92,5 +92,11 @@ use).
 
 By using this software, you also agree to the following licenses:
 https://github.com/CompVis/stable-diffusion/blob/main/LICENSE
+https://github.com/sd-webui/stable-diffusion-webui/blob/master/LICENSE
+https://github.com/invoke-ai/InvokeAI/blob/main/LICENSE
+https://github.com/cszn/BSRGAN/blob/main/LICENSE
+https://github.com/sczhou/CodeFormer/blob/master/LICENSE
 https://github.com/TencentARC/GFPGAN/blob/master/LICENSE
 https://github.com/xinntao/Real-ESRGAN/blob/master/LICENSE
+https://github.com/xinntao/ESRGAN/blob/master/LICENSE
+https://github.com/cszn/SCUNet/blob/main/LICENSE

README.md

@@ -23,16 +23,16 @@ This repository provides multiple UIs for you to play around with stable diffusi
 | ---------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- |
 | ![](https://user-images.githubusercontent.com/24505302/189541954-46afd772-d0c8-4005-874c-e2eca40c02f2.jpg) | ![](https://user-images.githubusercontent.com/24505302/189541956-5b528de7-1b5d-479f-a1db-d3f5a53afc59.jpg) | ![](https://user-images.githubusercontent.com/24505302/189541957-cf78b352-a071-486d-8889-f26952779a61.jpg) |
 
-### hlky
+### hlky (sd-webui / sygil-webui)
 
-[hlky's fork](https://github.com/hlky/stable-diffusion-webui) is one of the most popular UIs, with many features:
+[hlky's fork](https://github.com/Sygil-Dev/sygil-webui) is one of the most popular UIs, with many features:
 
 - Text to image, with many samplers
 - Image to image, with masking, cropping, in-painting, variations.
 - GFPGAN, RealESRGAN, LDSR, GoBig, GoLatent
 - Loopback, prompt weighting
 - 6GB or even 4GB GPU support!
-- [Full feature list here](https://github.com/sd-webui/stable-diffusion-webui/blob/master/README.md)
+- [Full feature list here](https://github.com/Sygil-Dev/sygil-webui/blob/master/README.md)
 
 Screenshots:
 
@@ -42,7 +42,7 @@ Screenshots:
 
 
 
-### lstein
+### lstein (InvokeAI)
 
 [lstein's fork](https://github.com/invoke-ai/InvokeAI) is one of the earliest with a wonderful WebUI.
 - Text to image, with many samplers
@@ -61,7 +61,9 @@ Visit the wiki for [Setup](https://github.com/AbdBarho/stable-diffusion-webui-do
 
 ## Contributing
 
-Contributions are welcome! create an issue first of what you want to contribute (before you implement anything) so we can talk about it.
+Contributions are welcome!
+
+### **Create an issue first of what you want to contribute (before you implement anything)**
 
 ## Disclaimer
 

docker-compose.yml

@@ -27,6 +27,7 @@ services:
     <<: *base_service
     profiles: ["hlky"]
     build: ./services/hlky/
+    image: sd-hlky:4
     environment:
       - CLI_ARGS=--optimized-turbo
       - USE_STREAMLIT=0
@@ -35,6 +36,7 @@ services:
     <<: *base_service
     profiles: ["auto"]
     build: ./services/AUTOMATIC1111
+    image: sd-auto:7
     environment:
       - CLI_ARGS=--allow-code --medvram --xformers
 
@@ -49,6 +51,7 @@ services:
     <<: *base_service
     profiles: ["lstein"]
     build: ./services/lstein/
+    image: sd-lstein:4
     environment:
       - PRELOAD=true
       - CLI_ARGS=

services/AUTOMATIC1111/Dockerfile

@@ -4,26 +4,29 @@ FROM alpine/git:2.36.2 as download
 
 SHELL ["/bin/sh", "-ceuxo", "pipefail"]
 
-RUN git clone https://github.com/CompVis/stable-diffusion.git repositories/stable-diffusion && cd repositories/stable-diffusion && git reset --hard 69ae4b35e0a0f6ee1af8bb9a5d0016ccb27e36dc
-
-RUN git clone https://github.com/sczhou/CodeFormer.git repositories/CodeFormer && cd repositories/CodeFormer && git reset --hard c5b4593074ba6214284d6acd5f1719b6c5d739af
-RUN git clone https://github.com/salesforce/BLIP.git repositories/BLIP && cd repositories/BLIP && git reset --hard 48211a1594f1321b00f14c9f7a5b4813144b2fb9
-
 RUN <<EOF
-# because taming-transformers is huge
-git config --global http.postBuffer 1048576000
-git clone https://github.com/CompVis/taming-transformers.git repositories/taming-transformers
-cd repositories/taming-transformers
-git reset --hard 24268930bf1dce879235a7fddd0b2355b84d7ea6
-rm -rf data assets
+cat <<'EOE' > /clone.sh
+mkdir -p repositories/"$1" && cd repositories/"$1" && git init && git remote add origin "$2" && git fetch origin "$3" --depth=1 && git reset --hard "$3" && rm -rf .git
+EOE
 EOF
 
-RUN git clone https://github.com/crowsonkb/k-diffusion.git repositories/k-diffusion && cd repositories/k-diffusion && git reset --hard f4e99857772fc3a126ba886aadf795a332774878
+RUN . /clone.sh taming-transformers https://github.com/CompVis/taming-transformers.git 24268930bf1dce879235a7fddd0b2355b84d7ea6 \
+  && rm -rf data assets **/*.ipynb
 
-FROM python:3.10-slim as xformers
-SHELL ["/bin/bash", "-ceuxo", "pipefail"]
-RUN pip install gdown
-RUN gdown https://drive.google.com/uc?id=1SqwicrLx1TrG_sbbEoIF_3TUHd4EYSmw -O /wheel.whl
+RUN . /clone.sh stable-diffusion https://github.com/CompVis/stable-diffusion.git 69ae4b35e0a0f6ee1af8bb9a5d0016ccb27e36dc \
+  && rm -rf assets data/**/*.png data/**/*.jpg data/**/*.gif
+
+RUN . /clone.sh CodeFormer https://github.com/sczhou/CodeFormer.git c5b4593074ba6214284d6acd5f1719b6c5d739af \
+  && rm -rf assets inputs
+
+RUN . /clone.sh BLIP https://github.com/salesforce/BLIP.git 48211a1594f1321b00f14c9f7a5b4813144b2fb9
+RUN . /clone.sh k-diffusion https://github.com/crowsonkb/k-diffusion.git f4e99857772fc3a126ba886aadf795a332774878
+RUN . /clone.sh clip-interrogator https://github.com/pharmapsychotic/clip-interrogator 2486589f24165c8e3b303f84e9dbbea318df83e8
+
+
+FROM alpine:3 as xformers
+RUN apk add aria2
+RUN aria2c --dir / --out wheel.whl 'https://github.com/AbdBarho/stable-diffusion-webui-docker/releases/download/2.1.0/xformers-0.0.14.dev0-cp310-cp310-linux_x86_64.whl'
 
 FROM python:3.10-slim
 
@@ -33,55 +36,55 @@ ENV DEBIAN_FRONTEND=noninteractive PIP_PREFER_BINARY=1 PIP_NO_CACHE_DIR=1
 
 RUN pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 --extra-index-url https://download.pytorch.org/whl/cu113
 
-RUN apt-get update && apt install fonts-dejavu-core rsync git -y && apt-get clean
+RUN apt-get update && apt install fonts-dejavu-core rsync git jq moreutils -y && apt-get clean
 
 
 RUN <<EOF
 git clone https://github.com/AUTOMATIC1111/stable-diffusion-webui.git
 cd stable-diffusion-webui
-git reset --hard 1eb588cbf19924333b88beaa1ac0041904966640
+git reset --hard d885a4a57b72152745ca76192ef1bdda29e6461d
 pip install -r requirements_versions.txt
 EOF
 
-ENV ROOT=/stable-diffusion-webui \
-  WORKDIR=/stable-diffusion-webui/repositories/stable-diffusion
+
+COPY --from=xformers /wheel.whl xformers-0.0.14.dev0-cp310-cp310-linux_x86_64.whl
+RUN pip install xformers-0.0.14.dev0-cp310-cp310-linux_x86_64.whl && rm xformers-0.0.14.dev0-cp310-cp310-linux_x86_64.whl
+
+
+ENV ROOT=/stable-diffusion-webui
 
 
 COPY --from=download /git/ ${ROOT}
+RUN mkdir ${ROOT}/interrogate && cp ${ROOT}/repositories/clip-interrogator/data/* ${ROOT}/interrogate
 RUN pip install --prefer-binary --no-cache-dir -r ${ROOT}/repositories/CodeFormer/requirements.txt
 
-# TODO: move to top
-RUN apt-get install jq moreutils -y
-
 
 # Note: don't update the sha of previous versions because the install will take forever
 # instead, update the repo state in a later step
 
-ARG SHA=36a0ba357ab0742c3c4a28437b68fb29a235afbe
+ARG SHA=35c45df28b303a05d56a13cb56d4046f08cf8c25
 RUN <<EOF
 cd stable-diffusion-webui
-git pull --rebase
+git fetch
 git reset --hard ${SHA}
-pip install --prefer-binary --no-cache-dir -r requirements_versions.txt
+pip install -r requirements_versions.txt
 EOF
 
-RUN pip install --prefer-binary --no-cache-dir opencv-python-headless \
+RUN pip install opencv-python-headless \
   git+https://github.com/TencentARC/GFPGAN.git@8d2447a2d918f8eba5a4a01463fd48e45126a379 \
   git+https://github.com/openai/CLIP.git@d50d76daa670286dd6cacf3bcd80b5e4823fc8e1 \
-  pyngrok
+  pyngrok \
+  gradio==3.4.1
+# gradio because #173 & https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/2858
 
-COPY --from=xformers /wheel.whl xformers-0.0.14.dev0-cp310-cp310-linux_x86_64.whl
-RUN pip install xformers-0.0.14.dev0-cp310-cp310-linux_x86_64.whl
 
 COPY . /docker
-RUN <<EOF
-chmod +x /docker/mount.sh && python3 /docker/info.py ${ROOT}/modules/ui.py
-EOF
 
+RUN chmod +x /docker/entrypoint.sh && python3 /docker/info.py ${ROOT}/modules/ui.py
 
+WORKDIR ${ROOT}/repositories/stable-diffusion
 ENV CLI_ARGS=""
-WORKDIR ${WORKDIR}
 EXPOSE 7860
+ENTRYPOINT ["/docker/entrypoint.sh"]
 # run, -u to not buffer stdout / stderr
-CMD /docker/mount.sh && \
-  python3 -u ../../webui.py --listen --port 7860 --ckpt-dir ${ROOT}/models/Stable-diffusion ${CLI_ARGS}
+CMD python3 -u ../../webui.py --listen --port 7860 --ckpt-dir ${ROOT}/models/Stable-diffusion --theme dark ${CLI_ARGS}

services/AUTOMATIC1111/entrypoint.sh

@@ -2,10 +2,18 @@
 
 set -Eeuo pipefail
 
-mkdir -p /data/config/auto/
+# TODO: move all mkdir -p ?
+mkdir -p /data/config/auto/scripts/
 cp -n /docker/config.json /data/config/auto/config.json
 jq '. * input' /data/config/auto/config.json /docker/config.json | sponge /data/config/auto/config.json
 
+if [ ! -f /data/config/auto/ui-config.json ]; then
+  echo '{}' >/data/config/auto/ui-config.json
+fi
+
+# copy scripts, we cannot just mount the directory because it will override the already provided scripts in the repo
+cp -rfT /data/config/auto/scripts/ "${ROOT}/scripts"
+
 declare -A MOUNTS
 
 MOUNTS["/root/.cache"]="/data/.cache"
@@ -24,6 +32,8 @@ MOUNTS["${ROOT}/models/hypernetworks"]="/data/Hypernetworks"
 
 MOUNTS["${ROOT}/embeddings"]="/data/embeddings"
 MOUNTS["${ROOT}/config.json"]="/data/config/auto/config.json"
+MOUNTS["${ROOT}/ui-config.json"]="/data/config/auto/ui-config.json"
+MOUNTS["${ROOT}/extensions"]="/data/config/auto/extensions"
 
 # extra hacks
 MOUNTS["${ROOT}/repositories/CodeFormer/weights/facelib"]="/data/.cache"
@@ -41,3 +51,12 @@ for to_path in "${!MOUNTS[@]}"; do
 done
 
 mkdir -p /output/saved /output/txt2img-images/ /output/img2img-images /output/extras-images/ /output/grids/ /output/txt2img-grids/ /output/img2img-grids/
+
+if [ -f "/data/config/auto/startup.sh" ]; then
+  pushd ${ROOT}
+  . /data/config/auto/startup.sh
+  popd
+fi
+
+
+exec "$@"

services/download/download.sh

@@ -4,21 +4,26 @@ set -Eeuo pipefail
 
 mkdir -p /data/.cache /data/StableDiffusion /data/Codeformer /data/GFPGAN /data/ESRGAN /data/BSRGAN /data/RealESRGAN /data/SwinIR /data/LDSR /data/ScuNET /data/embeddings
 
-cat <<EOF
-By using this software, you agree to the following licenses:
-https://github.com/CompVis/stable-diffusion/blob/main/LICENSE
-https://github.com/TencentARC/GFPGAN/blob/master/LICENSE
-https://github.com/xinntao/Real-ESRGAN/blob/master/LICENSE
-EOF
-
 echo "Downloading, this might take a while..."
 
-aria2c --input-file /docker/links.txt --dir /data --continue
+aria2c --disable-ipv6 --input-file /docker/links.txt --dir /data --continue
 
 echo "Checking SHAs..."
 
 parallel --will-cite -a /docker/checksums.sha256 "echo -n {} | sha256sum -c"
 
-# fix potential permissions
-# TODO: need something better than this:
-# chmod -R 777 /data /output
+# aria2c already does hash check
+# cc6cb27103417325ff94f52b7a5d2dde45a7515b25c255d8e396c90014281516  /data/StableDiffusion/v1-5-pruned-emaonly.ckpt
+cat <<EOF
+By using this software, you agree to the following licenses:
+https://github.com/CompVis/stable-diffusion/blob/main/LICENSE
+https://github.com/AbdBarho/stable-diffusion-webui-docker/blob/master/LICENSE
+https://github.com/sd-webui/stable-diffusion-webui/blob/master/LICENSE
+https://github.com/invoke-ai/InvokeAI/blob/main/LICENSE
+https://github.com/cszn/BSRGAN/blob/main/LICENSE
+https://github.com/sczhou/CodeFormer/blob/master/LICENSE
+https://github.com/TencentARC/GFPGAN/blob/master/LICENSE
+https://github.com/xinntao/Real-ESRGAN/blob/master/LICENSE
+https://github.com/xinntao/ESRGAN/blob/master/LICENSE
+https://github.com/cszn/SCUNet/blob/main/LICENSE
+EOF

services/download/links.txt

@@ -1,4 +1,12 @@
-https://www.googleapis.com/storage/v1/b/aai-blog-files/o/sd-v1-4.ckpt?alt=media
+# 'check-integrity=true' is the only way aria2c won't fail if the file already exists
+magnet:?xt=urn:btih:2daef5b5f63a16a9af9169a529b1a773fc452637&dn=v1-5-pruned-emaonly.ckpt&tr=udp%3a%2f%2ftracker.opentrackr.org%3a1337%2fannounce&tr=udp%3a%2f%2f9.rarbg.com%3a2810%2fannounce&tr=udp%3a%2f%2ftracker.openbittorrent.com%3a6969%2fannounce&tr=udp%3a%2f%2fopentracker.i2p.rocks%3a6969%2fannounce&tr=https%3a%2f%2fopentracker.i2p.rocks%3a443%2fannounce&tr=http%3a%2f%2ftracker.openbittorrent.com%3a80%2fannounce&tr=udp%3a%2f%2ftracker.torrent.eu.org%3a451%2fannounce&tr=udp%3a%2f%2fopen.stealth.si%3a80%2fannounce&tr=udp%3a%2f%2fvibe.sleepyinternetfun.xyz%3a1738%2fannounce&tr=udp%3a%2f%2ftracker2.dler.org%3a80%2fannounce&tr=udp%3a%2f%2ftracker1.bt.moack.co.kr%3a80%2fannounce&tr=udp%3a%2f%2ftracker.zemoj.com%3a6969%2fannounce&tr=udp%3a%2f%2ftracker.tiny-vps.com%3a6969%2fannounce&tr=udp%3a%2f%2ftracker.theoks.net%3a6969%2fannounce&tr=udp%3a%2f%2ftracker.publictracker.xyz%3a6969%2fannounce&tr=udp%3a%2f%2ftracker.monitorit4.me%3a6969%2fannounce&tr=udp%3a%2f%2ftracker.moeking.me%3a6969%2fannounce&tr=udp%3a%2f%2ftracker.lelux.fi%3a6969%2fannounce&tr=udp%3a%2f%2ftracker.dler.org%3a6969%2fannounce&tr=udp%3a%2f%2ftracker.army%3a6969%2fannounce
+  select-file=1
+  index-out=1=StableDiffusion/v1-5-pruned-emaonly.ckpt
+  follow-torrent=mem
+  follow-metalink=mem
+  seed-time=0
+  check-integrity=true
+https://drive.yerf.org/wl/?id=EBfTrmcCCUAGaQBXVIj5lJmEhjoP1tgl&mode=grid&download=1
   out=StableDiffusion/model.ckpt
 https://github.com/TencentARC/GFPGAN/releases/download/v1.3.4/GFPGANv1.4.pth
   out=GFPGAN/GFPGANv1.4.pth

services/hlky/Dockerfile

@@ -12,17 +12,20 @@ RUN conda install pytorch==1.11.0 torchvision==0.12.0 cudatoolkit=11.3 -c pytorc
 RUN apt-get update && apt install fonts-dejavu-core rsync gcc -y && apt-get clean
 
 
+ENV PIP_PREFER_BINARY=1 PIP_NO_CACHE_DIR=1
+
 RUN <<EOF
 git config --global http.postBuffer 1048576000
-git clone https://github.com/sd-webui/stable-diffusion-webui.git stable-diffusion
+git clone https://github.com/Sygil-Dev/sygil-webui.git stable-diffusion
 cd stable-diffusion
-git reset --hard 1a9c053cb7b6832695771db2555c0adc9b41e95f
+git reset --hard 091520bed06f913c9f432f9f47ccbe22b46068d7
 conda env update --file environment.yaml -n base
 conda clean -a -y
 EOF
 
+RUN apt-get update && apt install libsndfile1 ffmpeg -y && apt-get clean
 
-ARG BRANCH=dev SHA=bd57d22f2e0ec7eed084bdf2a4c0dcc0ed745daf
+ARG BRANCH=master SHA=091520bed06f913c9f432f9f47ccbe22b46068d7
 RUN <<EOF
 cd stable-diffusion
 git fetch

services/hlky/mount.sh

@@ -16,6 +16,7 @@ MOUNTS["${PWD}/models/custom"]=/data/StableDiffusion
 # hack
 MOUNTS["${PWD}/models/gfpgan/GFPGANv1.3.pth"]=/data/GFPGAN/GFPGANv1.4.pth
 MOUNTS["${PWD}/models/gfpgan/GFPGANv1.4.pth"]=/data/GFPGAN/GFPGANv1.4.pth
+MOUNTS["${PWD}/gfpgan/weights"]=/data/.cache
 
 
 for to_path in "${!MOUNTS[@]}"; do

services/hlky/run.sh

@@ -6,5 +6,5 @@ echo "USE_STREAMLIT = ${USE_STREAMLIT}"
 if [ "${USE_STREAMLIT}" == "1" ]; then
   python -u -m streamlit run scripts/webui_streamlit.py
 else
-  python3 -u scripts/webui.py --outdir /output --ckpt /data/StableDiffusion/model.ckpt ${CLI_ARGS}
+  python3 -u scripts/webui.py --outdir /output --ckpt /data/StableDiffusion/v1-5-pruned-emaonly.ckpt ${CLI_ARGS}
 fi

services/hlky/userconfig_streamlit.yaml

@@ -1,8 +1,9 @@
 general:
-  outdir: /outputs
-  default_model: "Stable Diffusion v1.4"
-  default_model_path: /data/StableDiffusion/model.ckpt
-  outdir_txt2img: /outputs/txt2img-samples
-  outdir_img2img: /outputs/img2img-samples
-  optimized: True
+  version: 1.24.6
+  outdir: /output
+  default_model: "Stable Diffusion v1.5"
+  default_model_path: /data/StableDiffusion/v1-5-pruned-emaonly.ckpt
+  outdir_txt2img: /output/txt2img-samples
+  outdir_img2img: /output/img2img-samples
+  outdir_img2txt: /output/img2txt
   optimized_turbo: True

services/lstein/Dockerfile

@@ -4,27 +4,25 @@ FROM continuumio/miniconda3:4.12.0
 
 SHELL ["/bin/bash", "-ceuxo", "pipefail"]
 
-ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBIAN_FRONTEND=noninteractive PIP_EXISTS_ACTION=w  PIP_PREFER_BINARY=1 PIP_NO_CACHE_DIR=1
 
 # now it requires python3.9
 RUN conda install python=3.9 && conda clean -a -y
 RUN conda install pytorch==1.11.0 torchvision==0.12.0 cudatoolkit=11.3 -c pytorch && conda clean -a -y
 
-RUN apt-get update && apt install fonts-dejavu-core rsync gcc -y && apt-get clean
-
-ENV PIP_EXISTS_ACTION=w
+RUN apt-get update && apt install fonts-dejavu-core -y && apt-get clean
 
 RUN <<EOF
 git clone https://github.com/invoke-ai/InvokeAI.git stable-diffusion
 cd stable-diffusion
-git reset --hard 79e79b78aaeedb49afcc795e0e00eebfdbedee96
+git reset --hard 986cc49d7a8441a70c709c04a55016a1c783f2ed
 git config --global http.postBuffer 1048576000
-conda env update --file environment.yml -n base
+# conda env update --file environment.yml -n base
 conda clean -a -y
 EOF
 
 
-ARG BRANCH=development SHA=fe2a2cfc8b6156805c420c026c5ef8bbb256899e
+ARG BRANCH=development SHA=fdf9b1c40c243d6806e787877e42ae4c115790ee
 RUN <<EOF
 cd stable-diffusion
 git fetch