1simple_SDXL_colab.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "T4",
      "authorship_tag": "ABX9TyMK0KJy+EltSl1KSwb1YZKX",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TheOneTrueGuy/DigitalDayDream/blob/main/1simple_SDXL_colab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "First the necessary installs and dependencies for using diffusers and transformers."
      ],
      "metadata": {
        "id": "sPlnpQubszgQ"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PBD5LWB2jNhc"
      },
      "outputs": [],
      "source": [
        "!pip install --upgrade diffusers transformers accelerate"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title You gotta use this too if you want easy save to your google drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "0FHobBfejZ55"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#!wget \"https://drive.google.com/file/d/196HuG4w9w_crcIjKaMGMxDme2R9UOS-F/view?usp=drive_link\" -O \"v2_lora_RollingClockwise.ckpt\""
      ],
      "metadata": {
        "id": "V2NkRnpbO_lX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Otherwise you can just download the final zip or .mp4 file."
      ],
      "metadata": {
        "id": "IELX3Q9swogw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title imagemagick gives us access to a simple, fast zoom function\n",
        "!apt install imagemagick\n",
        "# define zoom function using imagemagick convert (superior pixel and color handling, less pixelation than a simple resize)\n",
        "def zoom(imagename, zoomscale=1.1):\n",
        "  os.system(f\"convert {imagename} -distort SRT {zoomscale},0 -gravity center {imagename}\")\n",
        "\n"
      ],
      "metadata": {
        "id": "VLTLY7dxtJHW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# run this first then make anim in next cell\n",
        "import torch\n",
        "from diffusers import StableDiffusionXLImg2ImgPipeline\n",
        "from diffusers.utils import load_image\n",
        "#from diffusers import DiffusionPipeline\n",
        "import datetime as dt\n",
        "import time\n",
        "import os\n",
        "import random\n",
        "from PIL import ImageFont, ImageDraw, Image\n",
        "# fakesafety keeps the nsfw filter from returning black squares which will ruin our animations\n",
        "class FakeSafety():\n",
        "    def __call__(self, clip_input, images):\n",
        "        return (images, False)\n",
        "\n",
        "# assign simple generative pipe using the base function for use with LoRa layers.\n",
        "#if not using LoRa, comment this out and switch to next pipe\n",
        "pipe = StableDiffusionXLImg2ImgPipeline.from_pretrained(\n",
        "  \"stabilityai/stable-diffusion-xl-base-1.0\", torch_dtype=torch.float16\n",
        ")\n",
        "\n",
        "# this one is better if no LoRa is used, just remove the # from the next 3 lines.\n",
        "#pipe = StableDiffusionXLImg2ImgPipeline.from_pretrained(\n",
        "#  \"stabilityai/stable-diffusion-xl-refiner-1.0\", torch_dtype=torch.float16\n",
        "#)\n",
        "\n",
        "#experimental\n",
        "#pipe = StableDiffusionXLImg2ImgPipeline.from_pretrained(\n",
        "#  \"chillpixel/starlight-animated-sdxl\", torch_dtype=torch.float16\n",
        "#)\n",
        "#chillpixel/starlight-animated-sdxl\n",
        "\n",
        "# load lora moved to separate cell for optionality\n",
        "\n",
        "pipe = pipe.to(\"cuda\")\n",
        "pipe.safety_checker = FakeSafety()\n",
        "\n"
      ],
      "metadata": {
        "id": "W6TMuVpsjeDF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Load LoRa layer(s) this is optional, don't run if you just want to use default SDXL\n",
        "\n",
        "#https://replicate.com/jakedahn/sdxl-isometric-geology\n",
        "# v2_lora_RollingClockwise.ckpt\n",
        "\n",
        "pipe.load_lora_weights(\"goofyai/cyborg_style_xl\") # use with \"cyborg\" or \"android\" keyword\n",
        "#pipe.load_lora_weights(\"datamonet/Makima_Chainsaw_Man_LoRA\") #needs weight_name\n",
        "#pipe.load_lora_weights(\"\")\n",
        "\n",
        "#pipe.load_lora_weights(\"jakedahn/sdxl-isometric-geology\") #spare for swapping in new loras\n",
        "#pipe.load_lora_weights(\"\") #spare for swapping in new loras\n",
        "\n",
        "#FFusion/FFusionXL-LoRa-SDXL-Potion-Art-Engine\n",
        "\n",
        "# cadaeic/duelyst-landscape-lora\n",
        "# Duskfallcrew/the-crystal-exarch-15\n",
        "# datamonet/Makima_Chainsaw_Man_LoRA\n",
        "# the following might work with some param tweaking but I didn't have good luck with these LoRas\n",
        "# Norod78/sdxl-pumpkin-head-lora #not good\n",
        "# lora-library/tekakutli-dinosaurs #didn't work"
      ],
      "metadata": {
        "id": "AYlLnqhy9sjx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title upload start image 512x512 will be quick and look good\n",
        "\n",
        "import os\n",
        "\n",
        "from google.colab import files\n",
        "uploaded = files.upload()\n",
        "name=uploaded.keys()\n",
        "name = list(name)[0]\n",
        "print(name)\n"
      ],
      "metadata": {
        "id": "D5shShw9mkoy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n"
      ],
      "metadata": {
        "id": "1P0loMFnqCQM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "zip_name = \"easy_to_remember_name5\" #@param {type:\"string\"}\n",
        "negprompt = \" monochrome, dumb, flat, boring, ordinary\" #@param {type:\"string\"}\n",
        "prompt= \"Many happy robots dancing in the mechanical jungle of gears, axles and vine like chains stretching from telephone poles and silver pillars\" #@param {type:\"string\"}\n",
        "zoom_scale=0.99 #@param {type:\"number\"}\n",
        "# eventually we will add a math function to the zoom scale and/or keyframe triggers to change the rate.\n",
        "# notice! zoom scale above 1 like 1.05 will zoom in, below 1, like 0.995 will zoom out\n",
        "total_frames=300 #@param {type:\"number\"}\n",
        "\n",
        "# needs zip_name check to see if file already exists, append random number to prevent overwrite\n",
        "zip_name=zip_name+\".zip\"\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "outdir = dt.datetime.now().strftime(\"%Y%m%d%H%M\")\n",
        "# if outdir exists add 4 random digits to it:\n",
        "if os.path.exists(outdir):\n",
        "    # Generate 4 random digits\n",
        "    random_digits = str(random.randint(1000, 9999))\n",
        "\n",
        "    # Create a new output directory with the random digits appended\n",
        "    new_outdir = outdir + random_digits\n",
        "\n",
        "    os.mkdir(new_outdir)\n",
        "    outdir=new_outdir\n",
        "    print(\"Created new output directory:\", new_outdir)\n",
        "else:\n",
        "    os.system(f\"mkdir {outdir}\")\n",
        "print(\"date time now:\" + outdir)\n",
        "\n",
        "xz=0\n",
        "\n",
        "\n",
        "total=total_frames\n",
        "\n",
        "\n",
        "# main loop\n",
        "for xn in range(0,total,1):\n",
        "  if xn%50==49: torch.cuda.empty_cache()\n",
        "  init_image = load_image(name).convert(\"RGB\")#\n",
        "\n",
        "# low step numbers allow the image continuity to not get 'overwritten' by too many fine details in each new generation\n",
        "  image = pipe(\n",
        "               prompt,\n",
        "               num_inference_steps=10,\n",
        "               strength=0.81,\n",
        "               guidance_scale=7.34,\n",
        "               negative_prompt = negprompt ,\n",
        "               image=init_image\n",
        "               ).images[0]\n",
        "  outname= name\n",
        "\n",
        "  out2=outdir+\"/st-\" + str(xz).zfill(4) + \".png\"\n",
        "  image.save(out2) # save animation frame\n",
        "  xz+=1\n",
        "  image.save(outname)\n",
        "  # Check if zoom_scale is not equal to zero before calling the zoom function\n",
        "  if zoom_scale != 0:\n",
        "    zoom(outname, zoom_scale)\n",
        "\n",
        "  #next 2 lines apply motion distortion from moto*.txt motion files\n",
        "  # deleted lines to be replaced soon\n",
        "\n",
        "# all done, zip the files and save them\n",
        "os.system(f\"zip -r {zip_name} {outdir}\")\n",
        "os.system(f\"cp {zip_name}  /content/drive/MyDrive\")\n"
      ],
      "metadata": {
        "cellView": "code",
        "id": "6p8PGmUCkja8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# ffmpeg call to make .mp4 will go here soon"
      ],
      "metadata": {
        "id": "XlFJGY9GxrAn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#run this cell after starting cell(s) above if you don't want to burn\n",
        "#google compute credits unnecessarily.\n",
        "from google.colab import runtime\n",
        "runtime.unassign()"
      ],
      "metadata": {
        "id": "U-5X76bAzNTD"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}