Skip to content

Commit

Permalink
eyebrow
Browse files Browse the repository at this point in the history
  • Loading branch information
@yuyuyzl
yuyuyzl committed Jul 27, 2022
1 parent 8b291db commit 2b1277f
Show file tree
Hide file tree
Showing 4 changed files with 169 additions and 106 deletions.
3 changes: 2 additions & 1 deletion args.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,13 @@ def convert_to_byte(size):

parser = argparse.ArgumentParser()
parser.add_argument('--debug', action='store_true')
parser.add_argument('--eyebrow', action='store_true')
parser.add_argument('--extend_movement', type=float)
parser.add_argument('--input', type=str, default='cam')
parser.add_argument('--character', type=str, default='y')
parser.add_argument('--output_dir', type=str)
parser.add_argument('--output_webcam', type=str)
parser.add_argument('--output_size', type=str, default='256x256')
parser.add_argument('--output_size', type=str, default='512x512')
parser.add_argument('--model', type=str, default='standard_float')
parser.add_argument('--debug_input', action='store_true')
parser.add_argument('--mouse_input', type=str)
Expand Down
7 changes: 7 additions & 0 deletions launcher.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
'is_anime4k': False,
'is_alpha_split': False,
'is_bongo': False,
'is_eyebrow': False,
'cache_simplify': 1,
'cache_size': 1,
'model_type': 0,
Expand Down Expand Up @@ -89,6 +90,9 @@
ttk.Radiobutton(launcher, text='8GB+4GB', value=5, variable=cache_size).pack(fill='x', expand=True)

ttk.Label(launcher, text="Extra Options").pack(fill='x', expand=True)
is_eyebrow = tk.BooleanVar(value=args['is_eyebrow'])
ttk.Checkbutton(launcher, text='Eyebrow (iFM Only)', variable=is_eyebrow).pack(fill='x', expand=True)

is_extend_movement = tk.BooleanVar(value=args['is_extend_movement'])
ttk.Checkbutton(launcher, text='Extend Movement', variable=is_extend_movement).pack(fill='x', expand=True)

Expand Down Expand Up @@ -120,6 +124,7 @@ def launch():
'is_anime4k': is_anime4k.get(),
'is_alpha_split': is_alpha_split.get(),
'is_bongo': is_bongo.get(),
'is_eyebrow': is_eyebrow.get(),
'cache_simplify': cache_simplify.get(),
'cache_size': cache_size.get(),
'model_type': model_type.get(),
Expand Down Expand Up @@ -168,6 +173,8 @@ def launch():
run_args.append('1')
if args['is_bongo']:
run_args.append('--bongo')
if args['is_eyebrow']:
run_args.append('--eyebrow')
if args['cache_simplify'] is not None:
run_args.append('--simplify')
run_args.append(str(args['cache_simplify']))
Expand Down
132 changes: 72 additions & 60 deletions main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,10 +231,12 @@ def run(self):
print("Pretrained Model Loaded")


eyebrow_vector = torch.empty(1, 12,dtype=torch.half if args.model.endswith('half') else torch.float)
mouth_eye_vector = torch.empty(1, 27,dtype=torch.half if args.model.endswith('half') else torch.float)
pose_vector = torch.empty(1, 6,dtype=torch.half if args.model.endswith('half') else torch.float)

input_image = self.input_image.to(device)
eyebrow_vector = eyebrow_vector.to(device)
mouth_eye_vector = mouth_eye_vector.to(device)
pose_vector = pose_vector.to(device)

Expand Down Expand Up @@ -274,29 +276,29 @@ def run(self):
simplify_arr[ifm_converter.head_y_index] = 100
simplify_arr[ifm_converter.eye_surprised_left_index] = 10
simplify_arr[ifm_converter.eye_surprised_right_index] = 10
model_input[ifm_converter.eye_wink_left_index - 12] += model_input[
ifm_converter.eye_happy_wink_left_index - 12]
model_input[ifm_converter.eye_happy_wink_left_index - 12] = model_input[
ifm_converter.eye_wink_left_index - 12] / 2
model_input[ifm_converter.eye_wink_left_index - 12] = model_input[
ifm_converter.eye_wink_left_index - 12] / 2
model_input[ifm_converter.eye_wink_right_index - 12] += model_input[
ifm_converter.eye_happy_wink_right_index - 12]
model_input[ifm_converter.eye_happy_wink_right_index - 12] = model_input[
ifm_converter.eye_wink_right_index - 12] / 2
model_input[ifm_converter.eye_wink_right_index - 12] = model_input[
ifm_converter.eye_wink_right_index - 12] / 2

uosum = model_input[ifm_converter.mouth_uuu_index - 12] + \
model_input[ifm_converter.mouth_ooo_index - 12]
model_input[ifm_converter.mouth_ooo_index - 12] = uosum
model_input[ifm_converter.mouth_uuu_index - 12] = 0
is_open = (model_input[ifm_converter.mouth_aaa_index - 12] + model_input[
ifm_converter.mouth_iii_index - 12] + uosum) > 0
model_input[ifm_converter.mouth_lowered_corner_left_index - 12] = 0
model_input[ifm_converter.mouth_lowered_corner_right_index - 12] = 0
model_input[ifm_converter.mouth_raised_corner_left_index - 12] = 0.5 if is_open else 0
model_input[ifm_converter.mouth_raised_corner_right_index - 12] = 0.5 if is_open else 0
model_input[ifm_converter.eye_wink_left_index] += model_input[
ifm_converter.eye_happy_wink_left_index]
model_input[ifm_converter.eye_happy_wink_left_index] = model_input[
ifm_converter.eye_wink_left_index] / 2
model_input[ifm_converter.eye_wink_left_index] = model_input[
ifm_converter.eye_wink_left_index] / 2
model_input[ifm_converter.eye_wink_right_index] += model_input[
ifm_converter.eye_happy_wink_right_index]
model_input[ifm_converter.eye_happy_wink_right_index] = model_input[
ifm_converter.eye_wink_right_index] / 2
model_input[ifm_converter.eye_wink_right_index] = model_input[
ifm_converter.eye_wink_right_index] / 2

uosum = model_input[ifm_converter.mouth_uuu_index] + \
model_input[ifm_converter.mouth_ooo_index]
model_input[ifm_converter.mouth_ooo_index] = uosum
model_input[ifm_converter.mouth_uuu_index] = 0
is_open = (model_input[ifm_converter.mouth_aaa_index] + model_input[
ifm_converter.mouth_iii_index] + uosum) > 0
model_input[ifm_converter.mouth_lowered_corner_left_index] = 0
model_input[ifm_converter.mouth_lowered_corner_right_index] = 0
model_input[ifm_converter.mouth_raised_corner_left_index] = 0.5 if is_open else 0
model_input[ifm_converter.mouth_raised_corner_right_index] = 0.5 if is_open else 0
simplify_arr[ifm_converter.mouth_lowered_corner_left_index] = 0
simplify_arr[ifm_converter.mouth_lowered_corner_right_index] = 0
simplify_arr[ifm_converter.mouth_raised_corner_left_index] = 0
Expand All @@ -312,8 +314,8 @@ def run(self):
simplify_arr[ifm_converter.head_x_index] = 50
simplify_arr[ifm_converter.head_y_index] = 50
simplify_arr[ifm_converter.neck_z_index] = 100
model_input[ifm_converter.eye_raised_lower_eyelid_left_index - 12] = 0
model_input[ifm_converter.eye_raised_lower_eyelid_right_index - 12] = 0
model_input[ifm_converter.eye_raised_lower_eyelid_left_index] = 0
model_input[ifm_converter.eye_raised_lower_eyelid_right_index] = 0
simplify_arr[ifm_converter.iris_rotation_x_index] = 10
simplify_arr[ifm_converter.iris_rotation_y_index] = 10
simplify_arr[ifm_converter.eye_wink_left_index] = 24
Expand All @@ -322,43 +324,44 @@ def run(self):
simplify_arr[ifm_converter.eye_happy_wink_right_index] = 24
simplify_arr[ifm_converter.eye_surprised_left_index] = 8
simplify_arr[ifm_converter.eye_surprised_right_index] = 8
model_input[ifm_converter.eye_wink_left_index - 12] += model_input[
ifm_converter.eye_wink_right_index - 12]
model_input[ifm_converter.eye_wink_right_index - 12] = model_input[
ifm_converter.eye_wink_left_index - 12] / 2
model_input[ifm_converter.eye_wink_left_index - 12] = model_input[
ifm_converter.eye_wink_left_index - 12] / 2

model_input[ifm_converter.eye_surprised_left_index - 12] += model_input[
ifm_converter.eye_surprised_right_index - 12]
model_input[ifm_converter.eye_surprised_right_index - 12] = model_input[
ifm_converter.eye_surprised_left_index - 12] / 2
model_input[ifm_converter.eye_surprised_left_index - 12] = model_input[
ifm_converter.eye_surprised_left_index - 12] / 2

model_input[ifm_converter.eye_happy_wink_left_index - 12] += model_input[
ifm_converter.eye_happy_wink_right_index - 12]
model_input[ifm_converter.eye_happy_wink_right_index - 12] = model_input[
ifm_converter.eye_happy_wink_left_index - 12] / 2
model_input[ifm_converter.eye_happy_wink_left_index - 12] = model_input[
ifm_converter.eye_happy_wink_left_index - 12] / 2
model_input[ifm_converter.mouth_aaa_index - 12] = min(
model_input[ifm_converter.mouth_aaa_index - 12] +
model_input[ifm_converter.mouth_ooo_index - 12] / 2 +
model_input[ifm_converter.mouth_iii_index - 12] / 2 +
model_input[ifm_converter.mouth_uuu_index - 12] / 2, 1
model_input[ifm_converter.eye_wink_left_index] += model_input[
ifm_converter.eye_wink_right_index]
model_input[ifm_converter.eye_wink_right_index] = model_input[
ifm_converter.eye_wink_left_index] / 2
model_input[ifm_converter.eye_wink_left_index] = model_input[
ifm_converter.eye_wink_left_index] / 2

model_input[ifm_converter.eye_surprised_left_index] += model_input[
ifm_converter.eye_surprised_right_index]
model_input[ifm_converter.eye_surprised_right_index] = model_input[
ifm_converter.eye_surprised_left_index] / 2
model_input[ifm_converter.eye_surprised_left_index] = model_input[
ifm_converter.eye_surprised_left_index] / 2

model_input[ifm_converter.eye_happy_wink_left_index] += model_input[
ifm_converter.eye_happy_wink_right_index]
model_input[ifm_converter.eye_happy_wink_right_index] = model_input[
ifm_converter.eye_happy_wink_left_index] / 2
model_input[ifm_converter.eye_happy_wink_left_index] = model_input[
ifm_converter.eye_happy_wink_left_index] / 2
model_input[ifm_converter.mouth_aaa_index] = min(
model_input[ifm_converter.mouth_aaa_index] +
model_input[ifm_converter.mouth_ooo_index] / 2 +
model_input[ifm_converter.mouth_iii_index] / 2 +
model_input[ifm_converter.mouth_uuu_index] / 2, 1
)
model_input[ifm_converter.mouth_ooo_index - 12] = 0
model_input[ifm_converter.mouth_iii_index - 12] = 0
model_input[ifm_converter.mouth_uuu_index - 12] = 0
model_input[ifm_converter.mouth_ooo_index] = 0
model_input[ifm_converter.mouth_iii_index] = 0
model_input[ifm_converter.mouth_uuu_index] = 0
for i in range(4, args.simplify):
simplify_arr = [max(math.ceil(x * 0.8), 5) for x in simplify_arr]
for i in range(12, len(simplify_arr)):
for i in range(0, len(simplify_arr)):
if simplify_arr[i] > 0:
model_input[i - 12] = round(model_input[i - 12] * simplify_arr[i]) / simplify_arr[i]
model_input[i] = round(model_input[i] * simplify_arr[i]) / simplify_arr[i]
input_hash = hash(tuple(model_input))
cached = model_cache.get(input_hash)
tot += 1
eyebrow_vector_c = [0.0] * 12
mouth_eye_vector_c = [0.0] * 27
if cached is not None and hit_in_a_row < self.model_fps_number.value:
self.output_queue.put_nowait(cached)
Expand All @@ -369,15 +372,18 @@ def run(self):
hit_in_a_row = 0
if args.perf == 'model':
tic = time.perf_counter()
for i in range(12):
eyebrow_vector[0, i] = model_input[i]
eyebrow_vector_c[i] = model_input[i]
for i in range(27):
mouth_eye_vector[0, i] = model_input[i]
mouth_eye_vector_c[i] = model_input[i]
mouth_eye_vector[0, i] = model_input[i+12]
mouth_eye_vector_c[i] = model_input[i+12]
for i in range(6):
pose_vector[0, i] = model_input[i + 27]
pose_vector[0, i] = model_input[i + 27+12]
if model is None:
output_image = input_image
else:
output_image = model(input_image, mouth_eye_vector, pose_vector, mouth_eye_vector_c,
output_image = model(input_image, mouth_eye_vector, pose_vector, eyebrow_vector, mouth_eye_vector_c, eyebrow_vector_c,
self.gpu_cache_hit_ratio)
if args.perf == 'model':
torch.cuda.synchronize()
Expand Down Expand Up @@ -526,6 +532,7 @@ def main():
if args.perf == 'main':
tic = time.perf_counter()
if args.debug_input:
eyebrow_vector_c = [0.0] * 12
mouth_eye_vector_c = [0.0] * 27
pose_vector_c = [0.0] * 6

Expand Down Expand Up @@ -574,8 +581,11 @@ def main():
# - ifacialmocap_pose[EYE_LOOK_DOWN_RIGHT]
# + ifacialmocap_pose[EYE_LOOK_DOWN_LEFT]) / 2.0 / 0.75

eyebrow_vector_c = [0.0] * 12
mouth_eye_vector_c = [0.0] * 27
pose_vector_c = [0.0] * 6
for i in range(0, 12):
eyebrow_vector_c[i] = ifacialmocap_pose_converted[i]
for i in range(12, 39):
mouth_eye_vector_c[i - 12] = ifacialmocap_pose_converted[i]
for i in range(39, 42):
Expand All @@ -602,6 +612,7 @@ def main():
y_angle = mouse_data['y_angle']
z_angle = mouse_data['z_angle']

eyebrow_vector_c = [0.0] * 12
mouth_eye_vector_c = [0.0] * 27
pose_vector_c = [0.0] * 6

Expand Down Expand Up @@ -669,7 +680,8 @@ def main():
pose_vector_c[3] = pose_vector_c[1]
pose_vector_c[4] = pose_vector_c[2]

model_input_arr = mouth_eye_vector_c
model_input_arr = eyebrow_vector_c
model_input_arr.extend(mouth_eye_vector_c)
model_input_arr.extend(pose_vector_c)

model_process.input_queue.put_nowait(model_input_arr)
Expand Down
Loading

0 comments on commit 2b1277f

Please sign in to comment.