Auto Extensions (#176)

Closes #148 
Closes #172

.devscripts/chmod.sh

@@ -2,4 +2,4 @@
 
 set -Eeuo pipefail
 
-find . -name "*.sh" -exec git update-index --chmod=+x {} \;
+find services -name "*.sh" -exec git update-index --chmod=+x {} \;

.github/pull_request_template.md

@@ -1,3 +1,11 @@
+<!--
+Have you created an issue before opening a merge request???
+https://github.com/AbdBarho/stable-diffusion-webui-docker#contributing
+Please create one so we can discuss it, I don't want your effort to go to waste.
+-->
+
+Closes issue #
+
 ### Update versions
 
 - auto: https://github.com/AUTOMATIC1111/stable-diffusion-webui/commit/

.github/workflows/docker.yml

@@ -1,6 +1,12 @@
 name: Build Images
 
-on: [push]
+on:
+  push:
+    branches: master
+  pull_request:
+      paths:
+        - docker-compose.yml
+        - services
 
 jobs:
   build:

README.md

@@ -23,16 +23,16 @@ This repository provides multiple UIs for you to play around with stable diffusi
 | ---------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- |
 | ![](https://user-images.githubusercontent.com/24505302/189541954-46afd772-d0c8-4005-874c-e2eca40c02f2.jpg) | ![](https://user-images.githubusercontent.com/24505302/189541956-5b528de7-1b5d-479f-a1db-d3f5a53afc59.jpg) | ![](https://user-images.githubusercontent.com/24505302/189541957-cf78b352-a071-486d-8889-f26952779a61.jpg) |
 
-### hlky (sd-webui)
+### hlky (sd-webui / sygil-webui)
 
-[hlky's fork](https://github.com/sd-webui/stable-diffusion-webui) is one of the most popular UIs, with many features:
+[hlky's fork](https://github.com/Sygil-Dev/sygil-webui) is one of the most popular UIs, with many features:
 
 - Text to image, with many samplers
 - Image to image, with masking, cropping, in-painting, variations.
 - GFPGAN, RealESRGAN, LDSR, GoBig, GoLatent
 - Loopback, prompt weighting
 - 6GB or even 4GB GPU support!
-- [Full feature list here](https://github.com/sd-webui/stable-diffusion-webui/blob/master/README.md)
+- [Full feature list here](https://github.com/Sygil-Dev/sygil-webui/blob/master/README.md)
 
 Screenshots:
 
@@ -61,7 +61,9 @@ Visit the wiki for [Setup](https://github.com/AbdBarho/stable-diffusion-webui-do
 
 ## Contributing
 
-Contributions are welcome! create an issue first of what you want to contribute (before you implement anything) so we can talk about it.
+Contributions are welcome!
+
+### **Create an issue first of what you want to contribute (before you implement anything)**
 
 ## Disclaimer
 

docker-compose.yml

@@ -27,7 +27,7 @@ services:
     <<: *base_service
     profiles: ["hlky"]
     build: ./services/hlky/
-    image: sd-hlky:1
+    image: sd-hlky:4
     environment:
       - CLI_ARGS=--optimized-turbo
       - USE_STREAMLIT=0
@@ -36,7 +36,7 @@ services:
     <<: *base_service
     profiles: ["auto"]
     build: ./services/AUTOMATIC1111
-    image: sd-auto:1
+    image: sd-auto:7
     environment:
       - CLI_ARGS=--allow-code --medvram --xformers
 
@@ -51,7 +51,7 @@ services:
     <<: *base_service
     profiles: ["lstein"]
     build: ./services/lstein/
-    image: sd-lstein:1
+    image: sd-lstein:4
     environment:
       - PRELOAD=true
       - CLI_ARGS=

services/AUTOMATIC1111/Dockerfile

@@ -4,21 +4,24 @@ FROM alpine/git:2.36.2 as download
 
 SHELL ["/bin/sh", "-ceuxo", "pipefail"]
 
-RUN git clone https://github.com/CompVis/stable-diffusion.git repositories/stable-diffusion && cd repositories/stable-diffusion && git reset --hard 69ae4b35e0a0f6ee1af8bb9a5d0016ccb27e36dc
-
-RUN git clone https://github.com/sczhou/CodeFormer.git repositories/CodeFormer && cd repositories/CodeFormer && git reset --hard c5b4593074ba6214284d6acd5f1719b6c5d739af
-RUN git clone https://github.com/salesforce/BLIP.git repositories/BLIP && cd repositories/BLIP && git reset --hard 48211a1594f1321b00f14c9f7a5b4813144b2fb9
-
 RUN <<EOF
-# because taming-transformers is huge
-git config --global http.postBuffer 1048576000
-git clone https://github.com/CompVis/taming-transformers.git repositories/taming-transformers
-cd repositories/taming-transformers
-git reset --hard 24268930bf1dce879235a7fddd0b2355b84d7ea6
-rm -rf data assets
+cat <<'EOE' > /clone.sh
+mkdir -p repositories/"$1" && cd repositories/"$1" && git init && git remote add origin "$2" && git fetch origin "$3" --depth=1 && git reset --hard "$3" && rm -rf .git
+EOE
 EOF
 
-RUN git clone https://github.com/crowsonkb/k-diffusion.git repositories/k-diffusion && cd repositories/k-diffusion && git reset --hard f4e99857772fc3a126ba886aadf795a332774878
+RUN . /clone.sh taming-transformers https://github.com/CompVis/taming-transformers.git 24268930bf1dce879235a7fddd0b2355b84d7ea6 \
+  && rm -rf data assets **/*.ipynb
+
+RUN . /clone.sh stable-diffusion https://github.com/CompVis/stable-diffusion.git 69ae4b35e0a0f6ee1af8bb9a5d0016ccb27e36dc \
+  && rm -rf assets data/**/*.png data/**/*.jpg data/**/*.gif
+
+RUN . /clone.sh CodeFormer https://github.com/sczhou/CodeFormer.git c5b4593074ba6214284d6acd5f1719b6c5d739af \
+  && rm -rf assets inputs
+
+RUN . /clone.sh BLIP https://github.com/salesforce/BLIP.git 48211a1594f1321b00f14c9f7a5b4813144b2fb9
+RUN . /clone.sh k-diffusion https://github.com/crowsonkb/k-diffusion.git f4e99857772fc3a126ba886aadf795a332774878
+RUN . /clone.sh clip-interrogator https://github.com/pharmapsychotic/clip-interrogator 2486589f24165c8e3b303f84e9dbbea318df83e8
 
 
 FROM alpine:3 as xformers
@@ -33,34 +36,36 @@ ENV DEBIAN_FRONTEND=noninteractive PIP_PREFER_BINARY=1 PIP_NO_CACHE_DIR=1
 
 RUN pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 --extra-index-url https://download.pytorch.org/whl/cu113
 
-RUN apt-get update && apt install fonts-dejavu-core rsync git -y && apt-get clean
+RUN apt-get update && apt install fonts-dejavu-core rsync git jq moreutils -y && apt-get clean
 
 
 RUN <<EOF
 git clone https://github.com/AUTOMATIC1111/stable-diffusion-webui.git
 cd stable-diffusion-webui
-git reset --hard 1eb588cbf19924333b88beaa1ac0041904966640
+git reset --hard d885a4a57b72152745ca76192ef1bdda29e6461d
 pip install -r requirements_versions.txt
 EOF
 
-ENV ROOT=/stable-diffusion-webui \
-  WORKDIR=/stable-diffusion-webui/repositories/stable-diffusion
+
+COPY --from=xformers /wheel.whl xformers-0.0.14.dev0-cp310-cp310-linux_x86_64.whl
+RUN pip install xformers-0.0.14.dev0-cp310-cp310-linux_x86_64.whl && rm xformers-0.0.14.dev0-cp310-cp310-linux_x86_64.whl
+
+
+ENV ROOT=/stable-diffusion-webui
 
 
 COPY --from=download /git/ ${ROOT}
+RUN mkdir ${ROOT}/interrogate && cp ${ROOT}/repositories/clip-interrogator/data/* ${ROOT}/interrogate
 RUN pip install --prefer-binary --no-cache-dir -r ${ROOT}/repositories/CodeFormer/requirements.txt
 
-# TODO: move to top
-RUN apt-get install jq moreutils -y
-
 
 # Note: don't update the sha of previous versions because the install will take forever
 # instead, update the repo state in a later step
 
-ARG SHA=f49c08ea566385db339c6628f65c3a121033f67c
+ARG SHA=35c45df28b303a05d56a13cb56d4046f08cf8c25
 RUN <<EOF
 cd stable-diffusion-webui
-git pull --rebase
+git fetch
 git reset --hard ${SHA}
 pip install -r requirements_versions.txt
 EOF
@@ -68,20 +73,18 @@ EOF
 RUN pip install opencv-python-headless \
   git+https://github.com/TencentARC/GFPGAN.git@8d2447a2d918f8eba5a4a01463fd48e45126a379 \
   git+https://github.com/openai/CLIP.git@d50d76daa670286dd6cacf3bcd80b5e4823fc8e1 \
-  pyngrok
+  pyngrok \
+  gradio==3.4.1
+# gradio because #173 & https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/2858
 
-COPY --from=xformers /wheel.whl xformers-0.0.14.dev0-cp310-cp310-linux_x86_64.whl
-RUN pip install xformers-0.0.14.dev0-cp310-cp310-linux_x86_64.whl
 
 COPY . /docker
-RUN <<EOF
-chmod +x /docker/mount.sh && python3 /docker/info.py ${ROOT}/modules/ui.py
-EOF
 
+RUN chmod +x /docker/entrypoint.sh && python3 /docker/info.py ${ROOT}/modules/ui.py
 
+WORKDIR ${ROOT}/repositories/stable-diffusion
 ENV CLI_ARGS=""
-WORKDIR ${WORKDIR}
 EXPOSE 7860
+ENTRYPOINT ["/docker/entrypoint.sh"]
 # run, -u to not buffer stdout / stderr
-CMD /docker/mount.sh && \
-  python3 -u ../../webui.py --listen --port 7860 --ckpt-dir ${ROOT}/models/Stable-diffusion --theme dark ${CLI_ARGS}
+CMD python3 -u ../../webui.py --listen --port 7860 --ckpt-dir ${ROOT}/models/Stable-diffusion --theme dark ${CLI_ARGS}

services/AUTOMATIC1111/entrypoint.sh

@@ -2,7 +2,8 @@
 
 set -Eeuo pipefail
 
-mkdir -p /data/config/auto/
+# TODO: move all mkdir -p ?
+mkdir -p /data/config/auto/scripts/
 cp -n /docker/config.json /data/config/auto/config.json
 jq '. * input' /data/config/auto/config.json /docker/config.json | sponge /data/config/auto/config.json
 
@@ -10,6 +11,9 @@ if [ ! -f /data/config/auto/ui-config.json ]; then
   echo '{}' >/data/config/auto/ui-config.json
 fi
 
+# copy scripts, we cannot just mount the directory because it will override the already provided scripts in the repo
+cp -rfT /data/config/auto/scripts/ "${ROOT}/scripts"
+
 declare -A MOUNTS
 
 MOUNTS["/root/.cache"]="/data/.cache"
@@ -29,6 +33,7 @@ MOUNTS["${ROOT}/models/hypernetworks"]="/data/Hypernetworks"
 MOUNTS["${ROOT}/embeddings"]="/data/embeddings"
 MOUNTS["${ROOT}/config.json"]="/data/config/auto/config.json"
 MOUNTS["${ROOT}/ui-config.json"]="/data/config/auto/ui-config.json"
+MOUNTS["${ROOT}/extensions"]="/data/config/auto/extensions"
 
 # extra hacks
 MOUNTS["${ROOT}/repositories/CodeFormer/weights/facelib"]="/data/.cache"
@@ -46,3 +51,12 @@ for to_path in "${!MOUNTS[@]}"; do
 done
 
 mkdir -p /output/saved /output/txt2img-images/ /output/img2img-images /output/extras-images/ /output/grids/ /output/txt2img-grids/ /output/img2img-grids/
+
+if [ -f "/data/config/auto/startup.sh" ]; then
+  pushd ${ROOT}
+  . /data/config/auto/startup.sh
+  popd
+fi
+
+
+exec "$@"

services/download/download.sh

@@ -6,7 +6,7 @@ mkdir -p /data/.cache /data/StableDiffusion /data/Codeformer /data/GFPGAN /data/
 
 echo "Downloading, this might take a while..."
 
-aria2c --input-file /docker/links.txt --dir /data --continue
+aria2c --disable-ipv6 --input-file /docker/links.txt --dir /data --continue
 
 echo "Checking SHAs..."
 

services/download/links.txt

@@ -1,10 +1,10 @@
+# 'check-integrity=true' is the only way aria2c won't fail if the file already exists
 magnet:?xt=urn:btih:2daef5b5f63a16a9af9169a529b1a773fc452637&dn=v1-5-pruned-emaonly.ckpt&tr=udp%3a%2f%2ftracker.opentrackr.org%3a1337%2fannounce&tr=udp%3a%2f%2f9.rarbg.com%3a2810%2fannounce&tr=udp%3a%2f%2ftracker.openbittorrent.com%3a6969%2fannounce&tr=udp%3a%2f%2fopentracker.i2p.rocks%3a6969%2fannounce&tr=https%3a%2f%2fopentracker.i2p.rocks%3a443%2fannounce&tr=http%3a%2f%2ftracker.openbittorrent.com%3a80%2fannounce&tr=udp%3a%2f%2ftracker.torrent.eu.org%3a451%2fannounce&tr=udp%3a%2f%2fopen.stealth.si%3a80%2fannounce&tr=udp%3a%2f%2fvibe.sleepyinternetfun.xyz%3a1738%2fannounce&tr=udp%3a%2f%2ftracker2.dler.org%3a80%2fannounce&tr=udp%3a%2f%2ftracker1.bt.moack.co.kr%3a80%2fannounce&tr=udp%3a%2f%2ftracker.zemoj.com%3a6969%2fannounce&tr=udp%3a%2f%2ftracker.tiny-vps.com%3a6969%2fannounce&tr=udp%3a%2f%2ftracker.theoks.net%3a6969%2fannounce&tr=udp%3a%2f%2ftracker.publictracker.xyz%3a6969%2fannounce&tr=udp%3a%2f%2ftracker.monitorit4.me%3a6969%2fannounce&tr=udp%3a%2f%2ftracker.moeking.me%3a6969%2fannounce&tr=udp%3a%2f%2ftracker.lelux.fi%3a6969%2fannounce&tr=udp%3a%2f%2ftracker.dler.org%3a6969%2fannounce&tr=udp%3a%2f%2ftracker.army%3a6969%2fannounce
   select-file=1
   index-out=1=StableDiffusion/v1-5-pruned-emaonly.ckpt
   follow-torrent=mem
   follow-metalink=mem
   seed-time=0
-  # this is the only way aria2c won't fail if the file already exists
   check-integrity=true
 https://drive.yerf.org/wl/?id=EBfTrmcCCUAGaQBXVIj5lJmEhjoP1tgl&mode=grid&download=1
   out=StableDiffusion/model.ckpt

services/hlky/Dockerfile

@@ -12,17 +12,20 @@ RUN conda install pytorch==1.11.0 torchvision==0.12.0 cudatoolkit=11.3 -c pytorc
 RUN apt-get update && apt install fonts-dejavu-core rsync gcc -y && apt-get clean
 
 
+ENV PIP_PREFER_BINARY=1 PIP_NO_CACHE_DIR=1
+
 RUN <<EOF
 git config --global http.postBuffer 1048576000
-git clone https://github.com/sd-webui/stable-diffusion-webui.git stable-diffusion
+git clone https://github.com/Sygil-Dev/sygil-webui.git stable-diffusion
 cd stable-diffusion
-git reset --hard 1a9c053cb7b6832695771db2555c0adc9b41e95f
+git reset --hard 091520bed06f913c9f432f9f47ccbe22b46068d7
 conda env update --file environment.yaml -n base
 conda clean -a -y
 EOF
 
+RUN apt-get update && apt install libsndfile1 ffmpeg -y && apt-get clean
 
-ARG BRANCH=dev SHA=8d1e42b9c50c747d056b0a98f3c2eb7652fb73a7
+ARG BRANCH=master SHA=091520bed06f913c9f432f9f47ccbe22b46068d7
 RUN <<EOF
 cd stable-diffusion
 git fetch

services/hlky/mount.sh

@@ -16,6 +16,7 @@ MOUNTS["${PWD}/models/custom"]=/data/StableDiffusion
 # hack
 MOUNTS["${PWD}/models/gfpgan/GFPGANv1.3.pth"]=/data/GFPGAN/GFPGANv1.4.pth
 MOUNTS["${PWD}/models/gfpgan/GFPGANv1.4.pth"]=/data/GFPGAN/GFPGANv1.4.pth
+MOUNTS["${PWD}/gfpgan/weights"]=/data/.cache
 
 
 for to_path in "${!MOUNTS[@]}"; do

services/hlky/userconfig_streamlit.yaml

@@ -1,5 +1,5 @@
 general:
-  version: 1.20.0
+  version: 1.24.6
   outdir: /output
   default_model: "Stable Diffusion v1.5"
   default_model_path: /data/StableDiffusion/v1-5-pruned-emaonly.ckpt

services/lstein/Dockerfile

@@ -4,27 +4,25 @@ FROM continuumio/miniconda3:4.12.0
 
 SHELL ["/bin/bash", "-ceuxo", "pipefail"]
 
-ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBIAN_FRONTEND=noninteractive PIP_EXISTS_ACTION=w  PIP_PREFER_BINARY=1 PIP_NO_CACHE_DIR=1
 
 # now it requires python3.9
 RUN conda install python=3.9 && conda clean -a -y
 RUN conda install pytorch==1.11.0 torchvision==0.12.0 cudatoolkit=11.3 -c pytorch && conda clean -a -y
 
-RUN apt-get update && apt install fonts-dejavu-core rsync gcc -y && apt-get clean
-
-ENV PIP_EXISTS_ACTION=w
+RUN apt-get update && apt install fonts-dejavu-core -y && apt-get clean
 
 RUN <<EOF
 git clone https://github.com/invoke-ai/InvokeAI.git stable-diffusion
 cd stable-diffusion
-git reset --hard 79e79b78aaeedb49afcc795e0e00eebfdbedee96
+git reset --hard 986cc49d7a8441a70c709c04a55016a1c783f2ed
 git config --global http.postBuffer 1048576000
-conda env update --file environment.yml -n base
+# conda env update --file environment.yml -n base
 conda clean -a -y
 EOF
 
 
-ARG BRANCH=development SHA=554445a985d970200095bbcb109273a49c462682
+ARG BRANCH=development SHA=fdf9b1c40c243d6806e787877e42ae4c115790ee
 RUN <<EOF
 cd stable-diffusion
 git fetch