-
Notifications
You must be signed in to change notification settings - Fork 253
/
main.cpp
303 lines (266 loc) · 9.82 KB
/
main.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
#include <fstream>
#include "cv.h"
#include "highgui.h"
#include <filesystem.hpp>
#include <filesystem/fstream.hpp>
#include "FaceServices2.h"
#include <sstream>
#include "DlibWrapper.h"
#include <string>
#include <iostream>
#include "H5Cpp.h"
using namespace std;
using namespace cv;
// Load landmark from file (68x2)
cv::Mat loadLM(const char* LMfile){
ifstream in_stream(LMfile);
if (!in_stream.is_open()) {
return cv::Mat();
}
std::vector<float> vals;
string line;
while (!in_stream.eof())
{
line.clear();
std::getline(in_stream, line);
if (line.size() == 0 || line.at(0) == '#')
continue;
std::istringstream iss(line);
float x, y;
if (!(iss >> x >> y)) { continue; }
vals.push_back(x);
vals.push_back(y);
}
int N = vals.size()/2;
cv::Mat lms(vals.size()/2,2,CV_32F);
for (int i=0;i<N;i++){
lms.at<float>(i,0) = vals[2*i];
lms.at<float>(i,1) = vals[2*i+1];
}
return lms;
}
// Load shape parameters from file (99x1)
cv::Mat loadWeight(const char* inputFile){
ifstream in_stream(inputFile);
if (!in_stream.is_open()) {
return cv::Mat();
}
std::vector<float> vals;
string line;
while (!in_stream.eof())
{
line.clear();
std::getline(in_stream, line);
std::istringstream iss(line);
double w;
if (!(iss >> w)) { continue; }
vals.push_back(w);
}
int N = vals.size();
cv::Mat out(N,1,CV_32F);
for (int i=0;i<N;i++){
out.at<float>(i,0) = vals[i];
}
return out;
}
// Get cropped image:
// Inputs:
// oriIm : original image
// oriLMs : original landmarks (68x2)
// Outputs:
// newLMs : landmarks of the cropped image
// return the cropped image
cv::Mat getCroppedIm(Mat& oriIm, cv::Mat &oriLMs, cv::Mat &newLMs)
{
float padding = 1.7;
newLMs = oriLMs.clone();
Mat xs = newLMs(Rect(0, 0, 1, oriLMs.rows));
Mat ys = newLMs(Rect(1, 0, 1, oriLMs.rows));
// get LM-tight bounding box
double min_x, max_x, min_y, max_y;
cv::minMaxLoc(xs, &min_x, &max_x);
cv::minMaxLoc(ys, &min_y, &max_y);
float width = max_x - min_x;
float height = max_y - min_y;
if (width < 5 || height < 5 || width*height < 100) {
std::cout << "-> Error: Input face is too small" << std::endl;
return cv::Mat();
}
// expand bounding box
int minCropX = max((int)(min_x-padding*width/3.0),0);
int minCropY = max((int)(min_y-padding*height/3.0),0);
int widthCrop = min((int)(width*(3+2*padding)/3.0f), oriIm.cols - minCropX - 1);
int heightCrop = min((int)(height*(3+2*padding)/3.0f), oriIm.rows - minCropY - 1);
if(widthCrop <= 0 || heightCrop <=0) return cv::Mat();
// normalize image size to get a stable pose estimation, assuming focal length 1000
double minRes = 250*250 * (3+2*padding)/5.0f;
double maxRes = 300*300 * (3+2*padding)/5.0f;
double scaling = 1;
if (widthCrop*heightCrop < minRes) scaling = std::sqrt(widthCrop*heightCrop/minRes);
else if (widthCrop*heightCrop > maxRes) scaling = std::sqrt(widthCrop*heightCrop/maxRes);
// first crop the image
cv::Mat display_image = oriIm(Rect((int)(minCropX), (int)(minCropY), (int)(widthCrop), (int)(heightCrop)));
// now scale it
if (scaling != 1)
cv::resize(display_image.clone(), display_image, Size(), 1/scaling, 1/scaling);
int nrows = display_image.rows/4 * 4;
int ncols = display_image.cols/4 * 4;
if (nrows != display_image.rows || ncols != display_image.cols){
display_image = display_image(Rect(0,0,ncols,nrows)).clone();
}
xs = (xs - minCropX)/scaling;
ys = (ys - minCropY)/scaling;
return display_image;
}
int main(int argc, char** argv)
{
char text[200];
if (argc < 5) {
printf("Usage: Visualize3D <imPath> <input 3D alpha> <BaselFace.dat path> <dlib path> [<LM path>]\n");
return -1;
}
char imPath[200];
char lmPath[200] = "";
if (argc > 5){
strcpy(lmPath,argv[5]);
}
int overlaidType = 1; // 0: Windows 1: Docked in the second column 2: Hidden
bool showInput = true; // Config the 1st column in the output: the input image (true), or the cropped one (false)
strcpy(imPath,argv[1]);
//BaselFace::load_BaselFace_data(argv[3]);
DlibWrapper dw(argv[4]);
cv::Mat alpha = loadWeight(argv[2]);
if (alpha.rows != 99){
std::cout << "-> Error: Invalid alpha input!" << std::endl;
return -1;
}
int outSize = 380; // Size of visualized image
// original image
cv::Mat oriImg = imread(imPath);
if( oriImg.empty() ) return 0;
FaceServices2 fservice(argv[3]);
cv::Scalar normTextColor(0,0,0);
cv::Scalar boldTextColor(0,0,255);
cv::Scalar labelTextColor(255,0,0);
cv::Mat vecR, vecT, exprWeights;
cv::Mat im1, im2, imOut, imOut2, zbuffer, imFinal;
cv::Mat lms_init, lms;
// Get landmarks on the original image
if (strlen(lmPath) > 0){
lms_init = loadLM(lmPath);
if (lms_init.rows != 68){
printf("Bad landmark input file with %d points!\n", lms_init.rows);
return -1;
}
}
else {
std::vector<cv::Mat> lms0 = dw.detectLM(oriImg);
if (lms0.size() == 0){
printf("No face is detected!\n");
return -1;
}
lms_init = lms0[0].clone();
}
// get cropped image & landmarks
cv::Mat croppedImg = getCroppedIm(oriImg, lms_init, lms);
// Scale to fit output size.
cv::Mat croppedImg1 = croppedImg.clone();
float scale = outSize/((float)croppedImg1.rows);
float scaleX = floor(croppedImg1.cols*scale)/((float)(croppedImg1.cols));
cv::resize(croppedImg1, croppedImg,cv::Size(croppedImg1.cols*scale,outSize));
lms(cv::Rect(0, 0, 1, 68)) *= scaleX;
lms(cv::Rect(1, 0, 1, 68)) *= scale;
int ncols = croppedImg.cols/4 * 4;
if (ncols != croppedImg.cols)
croppedImg = croppedImg(cv::Rect(0,0,ncols,outSize)).clone();
fservice.init(croppedImg.cols,croppedImg.rows,1000.0f*scale); // Scale also the focal length
// estimate pose + expression
double ti = (double)getTickCount();
fservice.estimatePoseExpr(croppedImg, lms, alpha, vecR, vecT, exprWeights);
ti = ((double)getTickCount() - ti)/getTickFrequency();
// render face shape overlaying on the input image
double ti2 = (double)getTickCount();
fservice.renderShape(croppedImg, alpha, vecR, vecT, exprWeights, im2, zbuffer); // render
fservice.mergeIm(&im2,croppedImg,zbuffer); // overlay
ti2 = ((double)getTickCount() - ti2)/getTickFrequency();
printf("** Pose+expr fitting: %.3f s\n", ti);
printf("** Visualization : %.3f s\n", ti2);
printf("*****************************************\n");
if (overlaidType == 0) {
imshow("Overlaid",im2);
cv::moveWindow("Overlaid", 40, 550);
}
cv::Mat overlaidIm = im2.clone();
cv::Mat noOverlaidIm = croppedImg.clone();
// Prepare animated image
//FaceServices2 fservice2;
fservice.init(outSize,outSize,1000.0f*scale);
vecR = cv::Mat::zeros(3,1,CV_32F)+ 0.00001;
vecT = cv::Mat::zeros(3,1,CV_32F)+ 0.00001;
vecT.at<float>(0,0) = 5.156312*scale;
vecT.at<float>(1,0) = 11.053386*scale;
vecT.at<float>(2,0) = -709.398865*scale;
int currFrame = -1;
cv::Mat animatedImg = cv::Mat::zeros(outSize,outSize,CV_8UC3);
// 1st column in the output
if (showInput) {
scale = outSize/((float)oriImg.rows);
cv::resize(oriImg, im1,cv::Size(oriImg.cols*scale,outSize));
}
else im1 = noOverlaidIm.clone();
// Instruction texts
cv::Mat imInstruct(32, im1.cols + outSize + (overlaidType == 1)*overlaidIm.cols, CV_8UC3, cv::Scalar(150, 150, 150));
cv::putText(imInstruct,string("Input"), cv::Point((im1.cols-40)/2,18), FONT_HERSHEY_TRIPLEX, 0.5, labelTextColor);
if (overlaidType == 1)
cv::putText(imInstruct,string("Overlaid"), cv::Point(im1.cols + (overlaidIm.cols-64)/2,18), FONT_HERSHEY_TRIPLEX, 0.5, labelTextColor);
cv::putText(imInstruct,string("3D Model"), cv::Point(im1.cols + (overlaidType == 1)*overlaidIm.cols + (outSize-64)/2,18), FONT_HERSHEY_TRIPLEX, 0.5, labelTextColor);
cv::Mat imInstruct2(32, im1.cols + outSize + (overlaidType == 1)*overlaidIm.cols, CV_8UC3, cv::Scalar(255, 255, 255));
cv::putText(imInstruct2,string("Press to quit"), cv::Point(20,18), FONT_HERSHEY_TRIPLEX, 0.5, normTextColor);
cv::putText(imInstruct2,string("ESC"), cv::Point(70,18), FONT_HERSHEY_TRIPLEX, 0.5, boldTextColor);
// animate
bool firstFrame = true;
bool useOverlaid = true;
for(;;)
{
// get next pose & render
fservice.nextMotion(currFrame, vecR, vecT, exprWeights);
im2 = fservice.renderShape(animatedImg, alpha, vecR, vecT, exprWeights);
// display
if (overlaidType != 1) {
hconcat(im1,im2,imOut);
vconcat(imOut,imInstruct2,imFinal);
imshow("Model", imFinal);
}
else {
if (useOverlaid) hconcat(im1,overlaidIm,imOut);
else hconcat(im1,noOverlaidIm,imOut);
hconcat(imOut,im2,imOut2);
vconcat(imOut2,imInstruct,imOut);
vconcat(imOut,imInstruct2,imFinal);
imshow("Model", imFinal);
}
if (firstFrame) {
cv::moveWindow("Model", 40, 20);
firstFrame = false;
}
// check input key
char ch = waitKey(15);
if(ch == 27 ) break; // stop capturing by pressing ESC
else if (ch == 't') { // turn on/off overlay
useOverlaid = !useOverlaid;
if (overlaidType != 1) {
if (useOverlaid) imshow("Overlaid",overlaidIm);
else imshow("Overlaid",noOverlaidIm);
}
else {
if (useOverlaid) hconcat(im1,overlaidIm,imOut);
else hconcat(im1,noOverlaidIm,imOut);
hconcat(imOut,im2,imOut2);
vconcat(imOut2,imInstruct,imOut);
vconcat(imOut,imInstruct2,imFinal);
imshow("Model", imFinal);
}
}
}
return 0;
}