0Stream_Diffusion.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "T4",
      "authorship_tag": "ABX9TyP596PLTlH8b5ln/itAsphm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TheOneTrueGuy/DigitalDayDream/blob/main/0Stream_Diffusion.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7dJJKUNrn5AO"
      },
      "outputs": [],
      "source": [
        "!pip3 install torch==2.1.0 torchvision==0.16.0 xformers --index-url https://download.pytorch.org/whl/cu121"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install git+https://github.com/cumulo-autumn/StreamDiffusion.git@main#egg=streamdiffusion\n",
        "# wait here for restart"
      ],
      "metadata": {
        "id": "qVr-QD4VoINF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!python -m streamdiffusion.tools.install-tensorrt\n",
        "!pip install accelerate"
      ],
      "metadata": {
        "id": "zPLAMA7FoMLl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title upload start image 512x512 will be quick and look good\n",
        "\n",
        "import os\n",
        "\n",
        "from google.colab import files\n",
        "uploaded = files.upload()\n",
        "name=uploaded.keys()\n",
        "name = list(name)[0]\n",
        "print(name)\n"
      ],
      "metadata": {
        "id": "mEDsBWCuo05N"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# some stuff that might be useful later\n",
        "#os.system(f\"cp {name} st.png\") #if you will be using name file above for source image\n",
        "#name=\"st-0449.png\"\n",
        "#import os\n",
        "#os.system(f\"cp {name} st.png\")\n",
        "#os.system(\"convert {name} -resize 512x512! st.png\")\n",
        "#!cp /content/drive/MyDrive/stub_materials/cyberpunkcity.png /content/st.png\n",
        "#os.system(\"convert GAEhi0qbQAAeopj.jpg -resize 512x512! st.png\")"
      ],
      "metadata": {
        "id": "1VcrQjzBo8yk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import numpy as np\n",
        "from diffusers import AutoencoderTiny, StableDiffusionPipeline\n",
        "from diffusers.utils import load_image\n",
        "\n",
        "from streamdiffusion import StreamDiffusion\n",
        "from streamdiffusion.image_utils import postprocess_image\n",
        "\n",
        "pipe = StableDiffusionPipeline.from_pretrained(\"KBlueLeaf/kohaku-v2.1\").to(\n",
        "    device=torch.device(\"cuda\"),\n",
        "    dtype=torch.float16,\n",
        ")\n",
        "\n",
        "# Wrap the pipeline in StreamDiffusion\n",
        "stream = StreamDiffusion(\n",
        "    pipe,\n",
        "    t_index_list=[32, 45],\n",
        "    torch_dtype=torch.float16,\n",
        ")\n",
        "\n",
        "# If the loaded model is not LCM, merge LCM\n",
        "stream.load_lcm_lora()\n",
        "stream.fuse_lora()\n",
        "# Use Tiny VAE for further acceleration\n",
        "stream.vae = AutoencoderTiny.from_pretrained(\"madebyollin/taesd\").to(device=pipe.device, dtype=pipe.dtype)\n",
        "# Enable acceleration\n",
        "pipe.enable_xformers_memory_efficient_attention()\n"
      ],
      "metadata": {
        "id": "HWcqDovFoQZc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title image to image\n",
        "import torch\n",
        "from diffusers import AutoencoderTiny, StableDiffusionPipeline\n",
        "from diffusers.utils import load_image\n",
        "\n",
        "from streamdiffusion import StreamDiffusion\n",
        "from streamdiffusion.image_utils import postprocess_image\n",
        "\n",
        "# You can load any models using diffuser's StableDiffusionPipeline\n",
        "pipe = StableDiffusionPipeline.from_pretrained(\"KBlueLeaf/kohaku-v2.1\").to(\n",
        "    device=torch.device(\"cuda\"),\n",
        "    dtype=torch.float16,\n",
        ")\n",
        "\n",
        "# Wrap the pipeline in StreamDiffusion\n",
        "stream = StreamDiffusion(\n",
        "    pipe,\n",
        "    t_index_list=[32, 45],\n",
        "    torch_dtype=torch.float16,\n",
        ")\n",
        "\n",
        "# If the loaded model is not LCM, merge LCM\n",
        "stream.load_lcm_lora()\n",
        "stream.fuse_lora()\n",
        "# Use Tiny VAE for further acceleration\n",
        "stream.vae = AutoencoderTiny.from_pretrained(\"madebyollin/taesd\").to(device=pipe.device, dtype=pipe.dtype)\n",
        "# Enable acceleration\n",
        "pipe.enable_xformers_memory_efficient_attention()\n",
        "\n",
        "\n",
        "prompt = \"1girl with dog hair, thick frame glasses\"\n",
        "# Prepare the stream\n",
        "stream.prepare(prompt)\n",
        "\n",
        "# Prepare image\n",
        "init_image = load_image(name).resize((512, 512))\n",
        "\n",
        "# Warmup >= len(t_index_list) x frame_buffer_size\n",
        "for _ in range(2):\n",
        "    stream(init_image)\n",
        "\n",
        "# Run the stream infinitely\n",
        "while True:\n",
        "    x_output = stream(init_image)\n",
        "    postprocess_image(x_output, output_type=\"pil\")[0].show()\n",
        "    input_response = input(\"Press Enter to continue or type 'stop' to exit: \")\n",
        "    if input_response == \"stop\":\n",
        "        break"
      ],
      "metadata": {
        "id": "yA9KTpT68Irx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title text to image\n",
        "\n",
        "import torch\n",
        "from diffusers import AutoencoderTiny, StableDiffusionPipeline\n",
        "\n",
        "from streamdiffusion import StreamDiffusion\n",
        "from streamdiffusion.image_utils import postprocess_image\n",
        "\n",
        "# You can load any models using diffuser's StableDiffusionPipeline\n",
        "pipe = StableDiffusionPipeline.from_pretrained(\"KBlueLeaf/kohaku-v2.1\").to(\n",
        "    device=torch.device(\"cuda\"),\n",
        "    dtype=torch.float16,\n",
        ")\n",
        "\n",
        "# Wrap the pipeline in StreamDiffusion\n",
        "# Requires more long steps (len(t_index_list)) in text2image\n",
        "# You recommend to use cfg_type=\"none\" when text2image\n",
        "stream = StreamDiffusion(\n",
        "    pipe,\n",
        "    t_index_list=[0, 16, 32, 45],\n",
        "    torch_dtype=torch.float16,\n",
        "    cfg_type=\"none\",\n",
        ")\n",
        "\n",
        "# If the loaded model is not LCM, merge LCM\n",
        "stream.load_lcm_lora()\n",
        "stream.fuse_lora()\n",
        "# Use Tiny VAE for further acceleration\n",
        "stream.vae = AutoencoderTiny.from_pretrained(\"madebyollin/taesd\").to(device=pipe.device, dtype=pipe.dtype)\n",
        "# Enable acceleration\n",
        "pipe.enable_xformers_memory_efficient_attention()\n",
        "\n",
        "\n",
        "prompt = \"1girl with dog hair, thick frame glasses\"\n",
        "# Prepare the stream\n",
        "stream.prepare(prompt)\n",
        "\n",
        "# Warmup >= len(t_index_list) x frame_buffer_size\n",
        "for _ in range(4):\n",
        "    stream()\n",
        "\n",
        "# Run the stream infinitely\n",
        "while True:\n",
        "    x_output = stream.txt2img()\n",
        "    postprocess_image(x_output, output_type=\"pil\")[0].show()\n",
        "    input_response = input(\"Press Enter to continue or type 'stop' to exit: \")\n",
        "    if input_response == \"stop\":\n",
        "        break"
      ],
      "metadata": {
        "id": "e-dZxMw2YdHA"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}