-
Notifications
You must be signed in to change notification settings - Fork 34
/
align_img.py
124 lines (97 loc) · 3.44 KB
/
align_img.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
import numpy as np
import cv2
from scipy.io import loadmat,savemat
from PIL import Image
#TODO: refractor, support get translation from lm 5 and translate lm 68, put this file outof deep3drecon
#calculating least square problem
def POS(xp,x):
npts = xp.shape[1]
A = np.zeros([2*npts,8])
A[0:2*npts-1:2,0:3] = x.transpose()
A[0:2*npts-1:2,3] = 1
A[1:2*npts:2,4:7] = x.transpose()
A[1:2*npts:2,7] = 1;
b = np.reshape(xp.transpose(),[2*npts,1])
k,_,_,_ = np.linalg.lstsq(A,b)
R1 = k[0:3]
R2 = k[4:7]
sTx = k[3]
sTy = k[7]
s = (np.linalg.norm(R1) + np.linalg.norm(R2))/2
t = np.stack([sTx,sTy],axis = 0)
return t,s
def process_img(img,lm,t,s,target_size = 224.):
w0,h0 = img.size
w = (w0/s*102).astype(np.int32)
h = (h0/s*102).astype(np.int32)
img = img.resize((w,h),resample = Image.BICUBIC)
left = (w/2 - target_size/2 + float((t[0] - w0/2)*102/s)).astype(np.int32)
right = left + target_size
up = (h/2 - target_size/2 + float((h0/2 - t[1])*102/s)).astype(np.int32)
below = up + target_size
img = img.crop((left,up,right,below))
img = np.array(img)
img = img[:,:,::-1] #RGBtoBGR
img = np.expand_dims(img,0)
lm = np.stack([lm[:,0] - t[0] + w0/2,lm[:,1] - t[1] + h0/2],axis = 1)/s*102
lm = lm - np.reshape(np.array([(w/2 - target_size/2),(h/2-target_size/2)]),[1,2])
return img,lm
def align(img, lm, lm3D):
'''
Feed in Opencv Image
given image, lm, and standard lm, align the image
'''
img = img[:,:,::-1]
img = Image.fromarray(img)
w0,h0 = img.size
lm = np.stack([lm[:,0],h0 - 1 - lm[:,1]], axis = 1)
# calculate translation and scale factors using 5 facial landmarks and standard landmarks of a 3D face
t,s = POS(lm.transpose(),lm3D.transpose())
# processing the image
img_new,lm_new = process_img(img,lm,t,s)
lm_new = np.stack([lm_new[:,0],223 - lm_new[:,1]], axis = 1)
trans_params = np.array([w0,h0,102.0/s,t[0],t[1]])
return img_new,lm_new,trans_params
def align_video(img_array, lm_array, lm3D):
t_list = []
s_list = []
w0,h0 = Image.fromarray(img_array[0]).size
for i in range(len(lm_array)):
lm = lm_array[i]
lm = np.stack([lm[:,0],h0 - 1 - lm[:,1]], axis = 1)
t,s = POS(lm.transpose(),lm3D.transpose())
t_list.append(t)
s_list.append(s)
s = np.mean(np.array(s_list))
img_list = []
lm_new_list = []
trans_list = []
for i in range(len(img_array)):
t = t_list[i]
img = img_array[i][:,:,::-1]
img = Image.fromarray(img)
img_new, lm_new = process_img(img, lm_array[i], t_list[i], s)
lm_new = np.stack([lm_new[:,0],223 - lm_new[:,1]], axis = 1)
trans_params = np.array([w0,h0,102.0/s,t[0],t[1]])
img_list.append(img_new[0])
lm_new_list.append(lm_new)
trans_list.append(trans_params)
return np.array(img_list), np.array(lm_new_list), np.array(trans_list)
def align_lm68(lm5_list, lm68_list, lm3D, w0, h0, target_size = 224.):
'''
batch wise lm and lm 68
'''
lm68_new_list = []
for i in range(len(lm5_list)):
lm5 = lm5_list[i]
lm68 = lm68_list[i]
lm5 = np.stack([lm5[:,0],h0 - 1 - lm5[:,1]], axis = 1)
lm68 = np.stack([lm68[:,0],h0 - 1 - lm68[:,1]], axis = 1)
t,s = POS(lm5.transpose(),lm3D.transpose())
w = (w0/s*102).astype(np.int32)
h = (h0/s*102).astype(np.int32)
lm68_new = np.stack([lm68[:,0] - t[0] + w0/2,lm68[:,1] - t[1] + h0/2],axis = 1)/s*102
lm68_new = lm68_new - np.reshape(np.array([(w/2 - target_size/2),(h/2-target_size/2)]),[1,2])
lm68_new = np.stack([lm68_new[:,0], 223 - lm68_new[:,1]], axis = 1)
lm68_new_list.append(lm68_new)
return np.array(lm68_new_list)