forked from quic/ai-hub-models
-
Notifications
You must be signed in to change notification settings - Fork 0
/
demo.py
171 lines (153 loc) · 5.63 KB
/
demo.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
# ---------------------------------------------------------------------
# Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# ---------------------------------------------------------------------
import argparse
import numpy as np
import qai_hub as hub
from diffusers import DPMSolverMultistepScheduler, UNet2DConditionModel
from PIL import Image
from transformers import CLIPTokenizer
from qai_hub_models.models.stable_diffusion_quantized.app import StableDiffusionApp
from qai_hub_models.models.stable_diffusion_quantized.model import (
MODEL_ASSET_VERSION,
MODEL_ID,
ClipVITTextEncoder,
Unet,
VAEDecoder,
)
from qai_hub_models.utils.args import add_output_dir_arg
from qai_hub_models.utils.base_model import BasePrecompiledModel
from qai_hub_models.utils.display import display_or_save_image
from qai_hub_models.utils.inference import HubModel, get_uploaded_precompiled_model
from qai_hub_models.utils.qai_hub_helpers import can_access_qualcomm_ai_hub
DEFAULT_DEMO_PROMPT = "spectacular view of northern lights from Alaska"
DEFAULT_DEVICE_NAME = "Samsung Galaxy S23 Ultra"
def _get_hub_model(
input_model: BasePrecompiledModel,
model_name: str,
ignore_cached_model: bool = False,
device_name=DEFAULT_DEVICE_NAME,
):
if not can_access_qualcomm_ai_hub():
raise RuntimeError(
"Stable-diffusion on-device demo requires access to QAI-Hub.\n"
"Please visit https://aihub.qualcomm.com/ and sign-up."
)
# Upload model
uploaded_model = get_uploaded_precompiled_model(
input_model.get_target_model_path(),
MODEL_ID,
MODEL_ASSET_VERSION,
model_name,
ignore_cached_model=ignore_cached_model,
)
inputs = list(input_model.get_input_spec().keys())
return HubModel(uploaded_model, inputs, hub.Device(name=device_name))
# Run Stable Diffuison end-to-end on a given prompt. The demo will output an
# AI-generated image based on the description in the prompt.
def main(is_test: bool = False):
parser = argparse.ArgumentParser()
parser.add_argument(
"--prompt",
default=DEFAULT_DEMO_PROMPT,
help="Prompt to generate image from.",
)
parser.add_argument(
"--num-steps",
default=5,
type=int,
help="The number of diffusion iteration steps (higher means better quality).",
)
parser.add_argument(
"--seed",
default=0,
type=int,
help="Random seed.",
)
add_output_dir_arg(parser)
parser.add_argument(
"--guidance-scale",
type=float,
default=7.5,
help="Strength of guidance (higher means more influence from prompt).",
)
parser.add_argument(
"--ignore-cached-model",
action="store_true",
help="Uploads model ignoring previously uploaded and cached model.",
)
parser.add_argument(
"--device-name",
type=str,
default=DEFAULT_DEVICE_NAME,
help="Device to run stable-diffusion demo on.",
)
args = parser.parse_args([] if is_test else None)
if not is_test:
print(f"\n{'-' * 100}")
print(
f"** Performing image generation on-device({args.device_name}) with Stable Diffusion **"
)
print()
print("Prompt:", args.prompt)
print("Number of steps:", args.num_steps)
print("Guidance scale:", args.guidance_scale)
print("Seed:", args.seed)
print()
print(
"Note: This reference demo uses significant amounts of memory and may take 4-5 minutes to run ** per step **."
)
print(f"{'-' * 100}\n")
print(f"Downloading model assets\n{'-' * 35}")
# Load target models
text_encoder = ClipVITTextEncoder.from_precompiled()
unet = Unet.from_precompiled()
vae_decoder = VAEDecoder.from_precompiled()
# Create three HubModel instances to prepare for on-device inference.
# This is similar to initializing PyTorch model to call forward method later.
# Instead of forward, we later submit inference_jobs on QAI-Hub for
# on-device evaluation.
print(f"Uploading model assets on QAI-Hub\n{'-' * 35}")
text_encoder = _get_hub_model(
text_encoder, "text_encoder", args.ignore_cached_model, args.device_name
)
unet = _get_hub_model(unet, "unet", args.ignore_cached_model, args.device_name)
vae_decoder = _get_hub_model(
vae_decoder, "vae_decoder", args.ignore_cached_model, args.device_name
)
# Create tokenizer, scheduler and time_embedding required
# for stable-diffusion pipeline.
tokenizer = CLIPTokenizer.from_pretrained(
"stabilityai/stable-diffusion-2-1-base", subfolder="tokenizer", revision="main"
)
scheduler = DPMSolverMultistepScheduler(
beta_start=0.00085,
beta_end=0.012,
beta_schedule="scaled_linear",
num_train_timesteps=1000,
)
time_embedding = UNet2DConditionModel.from_pretrained(
"runwayml/stable-diffusion-v1-5", subfolder="unet"
).time_embedding
# Load Application
app = StableDiffusionApp(
text_encoder=text_encoder,
vae_decoder=vae_decoder,
unet=unet,
tokenizer=tokenizer,
scheduler=scheduler,
time_embedding=time_embedding,
)
# Generate image
image = app.generate_image(
args.prompt,
num_steps=args.num_steps,
seed=args.seed,
guidance_scale=args.guidance_scale,
)
pil_img = Image.fromarray(np.round(image.numpy() * 255).astype(np.uint8)[0])
if not is_test:
display_or_save_image(pil_img, args.output_dir)
if __name__ == "__main__":
main()