Reconstruction image process

MyLibs/experiment.cpp

@@ -259,3 +259,71 @@ void ExperimentData::runBlueTest()
 	cout << "Experiment ended. " << endl;
 
 }
+
+bool fishAngleAnalysis_test(String fishVideoAddress, bool isGrey) {
+	VideoCapture capture(fishVideoAddress);
+	Point testTail;
+	Mat curImg;
+
+	namedWindow("output", CV_WINDOW_NORMAL);
+	//namedWindow("org", CV_WINDOW_NORMAL);
+	capture >> curImg;
+	//findFishHeadAndCenter(curImg);
+	//getchar();
+	for (int i = 0; i < 1400; i++) {
+		capture >> curImg;
+	}
+
+
+	int n = 0;
+	int checkPoint=0;
+	int boutStart = 0;
+	int predict;
+	for (int i = 0; i < 10000; i++) {
+
+		Mat grey;
+		Point fishTail = Point(-1, -1);
+		double fishAngle[10000];
+		capture >> curImg;
+
+		if (!isGrey) {
+			cvtColor(curImg, grey, CV_BGR2GRAY);
+			if (!fishAngleAnalysis(grey, fishHead, fishCenter, &fishTail, &fishAngle[i], threshold_val)) {
+				cout << "AngleAnalysis error!" << endl;
+				return false;
+			}
+		}
+		else {
+			if (!fishAngleAnalysis(curImg, fishHead, fishCenter, &fishTail, &fishAngle[i], threshold_val)) {
+				cout << "AngleAnalysis error!" << endl;
+				return false;
+			}
+		}
+
+
+		if (n == checkPoint) {
+			if (boutStart > 0) {
+				predict=predict_left(&fishAngle[boutStart]);
+				cout << "predict:" << predict << endl;
+				boutStart = 0;
+			}
+
+			if (fishAngle[i] > 0.2|| fishAngle[i] < -0.2) {
+				boutStart = i - 4;
+				checkPoint = checkPoint + 40;
+				cout << "Find bout!" << endl;
+			}
+		}
+		else
+			n++;
+
+		cout << "fishAngle:" << fishAngle[i] << endl;
+		circle(curImg, fishTail, 1, Scalar(255, 0, 0), -1);
+		circle(curImg, tailPt_a, 1, Scalar(255, 0, 0), -1);
+		circle(curImg, tailPt_b, 1, Scalar(255, 0, 0), -1);
+		circle(curImg, topEnd, 1, Scalar(255, 0, 0), -1);
+		imshow("output", curImg);
+		waitKey();
+	}
+	return true;
+}

MyLibs/experiment.h

@@ -71,6 +71,6 @@ class Experiment
 	FileWriter fileWriterObj;
 
 };
-
+bool fishAngleAnalysis_test(String fishVideoAddress, bool isGrey);
 
 #endif _GUARD_EXPERIMENT_H

MyLibs/fishAnalysis.cpp

@@ -228,7 +228,7 @@ void Arena::getImgFromVideo(cv::VideoCapture cap)
 	cap >> opencvImg;//TODO: test this usage
 }
 
-void Arena::alignImg(int deg2rotate) 
+void Arena::alignImg(int deg2rotate)
 {
 	Mat rotatedImg;
 	rotateImg(opencvImg, rotatedImg, deg2rotate); //TODO: write the implementation
@@ -296,7 +296,7 @@ bool Arena::findSingleFish()
 		int contourSize = contour.size();
 		if (contourSize < contourSizeThre)
 			continue; // skip this turn
-		
+
 		if (maxContourSize < contour.size())
 		{
 			maxContourSize = contour.size();
@@ -494,7 +494,7 @@ bool Fish::checkIfGiveShock(int sElapsed) {
 	if (head.x == -1) // invalid frame
 		return false;
 	if (idxCase) // patternIdx == 1, since 2 is already excluded
-	{ 
+	{
 		if (head.y < yDiv) // in non-CS area
 			shockOn = false;
 		else {
@@ -538,7 +538,7 @@ bool Fish::checkIfReversePattern(int sElapsed)
 	{
 		idxCase = !idxCase;
 		return true;
-	}	
+	}
 	else
 		return false;
 }
@@ -643,4 +643,82 @@ void rot90CW(Mat src, Mat dst)
 	transpose(temp, dst);
 }
 
+/*This function return a radian to describe the fishtailing motion */
+bool fishAngleAnalysis(Mat fishImg, Point fishHead, Point fishCenter, Point * fishTail_return, double* fishAngle,int threshold_val) {
+	//Find the contour of fish
+	Mat binaryzation;
+	double  max_val = 255, maxFishArea = 10000, minFishArea = 1000;
+	vector<vector<Point>> allContours, fishContours;
+	threshold(fishImg, binaryzation, threshold_val, max_val, CV_THRESH_BINARY);
+	findContours(binaryzation, allContours, CV_RETR_LIST, CHAIN_APPROX_NONE);
+	for (int i = 0; i < allContours.size(); i++) {
+		if (contourArea(allContours[i]) < maxFishArea && contourArea(allContours[i]) > minFishArea)
+			fishContours.push_back(allContours[i]);
+	}
+	if (fishContours.size() != 1) {
+		cout << "Can't find contour of fish!Area of all contours:";
+		for (int i = 0; i < allContours.size(); i++) {
+			cout << contourArea(allContours[i]) << ',';
+		}
+		cout << endl;
+		return false;
+	}
+
+	//Find the tail of fish
 
+	double Pt2center = norm(fishContours[0][0] - fishCenter);
+	topEnd = fishContours[0][0];
+
+	for (int i = 1; i < fishContours[0].size(); i++)
+	{
+		double curPt2center = norm(fishContours[0][i] - fishCenter);
+		if (Pt2center < curPt2center) {
+			topEnd = fishContours[0][i];
+			Pt2center = curPt2center;
+			//circle(fishImg, topEnd, 1, Scalar(255), -1);
+
+		}
+
+
+	}
+	Point tailAxis = topEnd - fishCenter;
+	tailPt_a = fishCenter + tailAxis * 9 / 10 + Point(tailAxis.y, -tailAxis.x)/4;
+	tailPt_b = fishCenter + tailAxis * 9 / 10 + Point(-tailAxis.y, tailAxis.x)/4;
+	vector<int> fishTail = findPtsLineIntersectContour(fishContours[0], tailPt_a, tailPt_b);
+
+	//Calculate the angle
+	Point fishHeadVector, fishTailVector;
+	fishHeadVector = fishCenter - fishHead;
+	fishTailVector = (fishContours[0][fishTail[0]]+ fishContours[0][fishTail[1]])/2 - fishCenter;
+	double sinfi;
+	sinfi = -(fishHeadVector.x * fishTailVector.y - fishTailVector.x * fishHeadVector.y) / (norm(fishHeadVector) * norm(fishTailVector));
+	*fishAngle = asin(sinfi);
+	*fishTail_return = (fishContours[0][fishTail[0]] + fishContours[0][fishTail[1]]) / 2;
+
+	return true;
+}
+
+int predict_left(double* boutStart) {
+	Py_Initialize();
+	if (Py_IsInitialized() == 0) {
+		cout << "Py_Initialize failed." << endl;
+	}
+	PyObject* pModule = PyImport_ImportModule("predict");
+	if (pModule == NULL)
+		cout << "Py_ImportModule failed." << endl;
+	PyObject * pFunc = PyObject_GetAttrString(pModule, "predict_left");
+	PyObject * PyList = PyList_New(40);
+	PyObject * ArgList = PyTuple_New(1);
+	for (int Index_i = 0; Index_i < PyList_Size(PyList); Index_i++) {
+		PyList_SetItem(PyList, Index_i, PyFloat_FromDouble(boutStart[Index_i]));
+	}
+	PyTuple_SetItem(ArgList, 0, PyList);
+	PyObject* pReturn = NULL;
+	pReturn = PyObject_CallObject(pFunc, ArgList);
+	int result;
+	PyArg_Parse(pReturn, "i", &result);
+	cout << "predict:" << result << endl;
+	Py_Finalize();
+	return result;
+
+}

MyLibs/fishAnalysis.h

@@ -33,6 +33,8 @@
 // Include user-defined libraries
 #include "errorHandling.h"
 
+#include <Python.h>
+
 /* Define related methods and properties for a single fish */
 // Write every frame updated info at here? No! Create another class in fileWriter class
 class Fish {
@@ -215,7 +217,7 @@ class FishAnalysis {
 				{ 223, 223, 588, 588 },
 				{ 223, 223, 588, 588 }
 			}; // TODO: make this variable private
-			
+
 			aIdx = 0;
 		}
 
@@ -227,7 +229,7 @@ class FishAnalysis {
 		void initialize(std::vector<int> numFishInArenas);
 
 		/* Process image from camera */
-		void preprocessImg(); 
+		void preprocessImg();
 
 		/* Get image from camera */
 		void getImgFromCamera(int width, int height, uint8_t* buffer);
@@ -281,7 +283,7 @@ class FishAnalysis {
 		int numArenas;
 		int aIdx; // index of arena to process
 private:
-		std::vector<std::vector<int>> yDivs; 
+		std::vector<std::vector<int>> yDivs;
 };
 
 
@@ -305,7 +307,9 @@ double getPt2LineDistance(cv::Point2f P, cv::Point2f A, cv::Point2f B);
 /* Find 2 intersection points of a line (AB) and contour */
 std::vector<int> findPtsLineIntersectContour(std::vector<cv::Point>& contour, cv::Point2f A, cv::Point2f B);
 
-
-
+/*This function return a radian to describe the fishtailing motion */
+bool fishAngleAnalysis(Mat fishImg, Point fishHead, Point fishCenter, Point * fishTail_return, double* fishAngle,int threshold_val);
+/*This function return the direction of motion(is or not left),which predicted from 40 radians*/
+int predict_left(double* boutStart);
 
 #endif // !_GUARD_FISHANALYSIS_H

MyLibs/predict.py

@@ -0,0 +1,14 @@
+import h2o
+import numpy as np
+import pandas as pd
+
+def predict_left(List):
+    PyList=List
+    #print(PyList)
+    df=pd.DataFrame(np.zeros((1,40)),columns=range(1,41))
+    df.iloc[0]=PyList
+    test_predict=h2o.mojo_predict_pandas(df, 'E:/autoML/left_true_0513_accuracy_09516/GBM_grid_1_AutoML_20190513_170806_model_7.zip')
+    print(test_predict)
+    return int(test_predict.iat[0,0])
+
+#def predict_right(List):

MyLibs/userInterface.cpp

@@ -57,13 +57,13 @@ void UserInterface::enquireDevice2use(std::istream& is)
 		{
 			devices2use[1] = 1;
 			enquirePattern2use(is);
-		}		
+		}
 		else if (!s.compare("3"))
 		{
 			devices2use[2] = 1;
 			enquireCameras2use(is);
 		}
-			
+
 		else
 		{
 			cout << "Invalid input! Please enter again." << endl;
@@ -87,7 +87,7 @@ void UserInterface::enquireCameras2use(std::istream& is)
 
 	for (int i = 0; i < numCameras; i++)
 		cameras2open[i] = stoi(tempStrVec[i]);
-	
+
 	for (int i = 0; i < cameras2open.size(); i++)
 	{
 		if (cameras2open[i])
@@ -227,7 +227,7 @@ void UserInterface::generateBasenames()
 string UserInterface::generateBasename(int idxFile)
 {
 	string baseName =
-		startTimeStr + "_" + "Arena" 
+		startTimeStr + "_" + "Arena"
 		+ to_string(arenaIDs[idxFile])
 		+ "_" + strainName + "_"
 		+ to_string(fishAge)+ "dpf_"
@@ -251,7 +251,7 @@ void showWelcomeMsg()
 	Copyright 2018 Wenbin Yang <[email protected]>
 	*/
 	cout << "Welcome to BLITZ (Behavioral Learning In The Zebrafish)." << endl
-		<< "This program is under GNU 3.0 License." << endl 
+		<< "This program is under GNU 3.0 License." << endl
 		<< "Most updated code and other resources can be found at " << endl
 		<< "https://github.com/Wenlab/BLITZ" << endl
 		<< "Please cite (Wenbin Yang et al., 2019) if you use any portion of this program." << endl
@@ -283,7 +283,7 @@ vector<string> getStrVecFromCMD(std::istream& is)
 	string inputStr;
 	getline(is, inputStr);
 	cout << endl; // separated with an empty line
-	
+
 
 	istringstream ss;
 	ss.clear();
@@ -315,3 +315,59 @@ string getCurDateTime()
 	timeStr = buffer;
 	return timeStr;
 }
+
+static void on_trackbar_setThreshold(int, void*) {
+	int max_val = 255;
+	Mat binaryzation = Mat::zeros(cur_img.size(), CV_8UC1);
+	threshold(cur_img, binaryzation, threshold_val, max_val, CV_THRESH_BINARY);
+
+	imshow("setThreshold", binaryzation);
+
+}
+
+bool setThreshold() {
+	namedWindow("setThreshold", CV_WINDOW_NORMAL);
+	createTrackbar("Threshold", "setThreshold", &threshold_val, 255, on_trackbar_setThreshold);
+	on_trackbar_setThreshold(threshold_val, 0);
+	cout<<"Press 'q' to exit."<<endl;
+	while (char(waitKey(1)) != 'q') {}
+	return true;
+}
+
+void on_mouse_findHeadAndCenter(int event, int x, int y, int flags, void* ustc) {
+	if (event == CV_EVENT_LBUTTONDBLCLK) {
+		fishEye1 = Point(x, y);
+		circle(cur_img, fishEye1, 2, Scalar(255, 0, 0, 0), CV_FILLED, CV_AA, 0);
+		imshow("findHeadAndCenter", cur_img);
+	}
+	else if (event == CV_EVENT_RBUTTONDBLCLK) {
+
+		fishEye2 = Point(x, y);
+		fishHead = (fishEye1 + fishEye2) / 2;
+		circle(cur_img, fishHead, 2, Scalar(0, 0, 0, 255), CV_FILLED, CV_AA, 0);
+		circle(cur_img, fishEye2, 2, Scalar(0, 255, 0, 0), CV_FILLED, CV_AA, 0);
+		imshow("findHeadAndCenter", cur_img);
+
+	}
+	else if (event == CV_EVENT_MBUTTONDOWN) {
+
+		fishCenter = Point(x, y);
+		circle(cur_img, fishCenter, 2, Scalar(0, 0, 255, 0), CV_FILLED, CV_AA, 0);
+		imshow("findHeadAndCenter", cur_img);
+		cout << "fishCenter:" << fishCenter.x << ',' << fishCenter.y << endl;
+		cout << "fishHead:" << fishHead.x << ',' << fishHead.y << endl;
+	}
+
+}
+
+bool findHeadAndCenter() {
+	namedWindow("findHeadAndCenter", CV_WINDOW_NORMAL);
+	imshow("findHeadAndCenter", cur_img);
+	setMouseCallback("findHeadAndCenter", on_mouse_findHeadAndCenter, 0);
+	cout<<"Double-click the left mouse button to select one eye."<<endl;
+	cout<<"Double-click the right mouse button to select another eye."<<endl;
+	cout<<"Click the middle mouse button to select center."<<endl;
+	cout<<"Press 'q' to exit."<<endl;
+	while (char(waitKey(1)) != 'q') {}
+	return true;
+}