Added implementation for Emojinator

examples/keras/Emojinator/CreateCSV.py

@@ -0,0 +1,21 @@
+from scipy.misc import imread
+import numpy as np
+import pandas as pd
+import os
+root = './gestures' # or ‘./test’ depending on for which the CSV is being created
+
+# go through each directory in the root folder given above
+for directory, subdirectories, files in os.walk(root):
+# go through each file in that directory
+    for file in files:
+    # read the image file and extract its pixels
+        print(file)
+        im = imread(os.path.join(directory,file))
+        value = im.flatten()
+# I renamed the folders containing digits to the contained digit itself. For example, digit_0 folder was renamed to 0.
+# so taking the 9th value of the folder gave the digit (i.e. "./train/8" ==> 9th value is 8), which was inserted into the first column of the dataset.
+        value = np.hstack((directory[8:],value))
+        df = pd.DataFrame(value).T
+        df = df.sample(frac=1) # shuffle the dataset
+        with open('train_foo.csv', 'a') as dataset:
+            df.to_csv(dataset, header=False, index=False)

examples/keras/Emojinator/CreateGest.py

@@ -0,0 +1,76 @@
+import cv2
+import numpy as np
+import os
+
+image_x, image_y = 50, 50
+
+cap = cv2.VideoCapture(0)
+fbag = cv2.createBackgroundSubtractorMOG2()
+
+def create_folder(folder_name):
+    if not os.path.exists(folder_name):
+        os.mkdir(folder_name)
+
+def main(g_id):
+    total_pics = 1200
+    cap = cv2.VideoCapture(0)
+    x, y, w, h = 300, 50, 350, 350
+
+    create_folder("gestures/" + str(g_id))
+    pic_no = 0
+    flag_start_capturing = False
+    frames = 0
+
+    while True:
+        ret, frame = cap.read()
+        frame = cv2.flip(frame, 1)
+        hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
+
+        mask2 = cv2.inRange(hsv, np.array([2, 50, 60]), np.array([25, 150, 255]))
+        res = cv2.bitwise_and(frame, frame, mask=mask2)
+        gray = cv2.cvtColor(res, cv2.COLOR_BGR2GRAY)
+        median = cv2.GaussianBlur(gray, (5, 5), 0)
+
+        kernel_square = np.ones((5, 5), np.uint8)
+        dilation = cv2.dilate(median, kernel_square, iterations=2)
+        opening=cv2.morphologyEx(dilation,cv2.MORPH_CLOSE,kernel_square)
+
+        ret, thresh = cv2.threshold(opening, 30, 255, cv2.THRESH_BINARY)
+        thresh = thresh[y:y + h, x:x + w]
+        contours = cv2.findContours(thresh.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)[1]
+
+        if len(contours) > 0:
+            contour = max(contours, key=cv2.contourArea)
+            if cv2.contourArea(contour) > 10000 and frames > 50:
+                x1, y1, w1, h1 = cv2.boundingRect(contour)
+                pic_no += 1
+                save_img = thresh[y1:y1 + h1, x1:x1 + w1]
+                if w1 > h1:
+                    save_img = cv2.copyMakeBorder(save_img, int((w1 - h1) / 2), int((w1 - h1) / 2), 0, 0,
+                                                  cv2.BORDER_CONSTANT, (0, 0, 0))
+                elif h1 > w1:
+                    save_img = cv2.copyMakeBorder(save_img, 0, 0, int((h1 - w1) / 2), int((h1 - w1) / 2),
+                                                  cv2.BORDER_CONSTANT, (0, 0, 0))
+                save_img = cv2.resize(save_img, (image_x, image_y))
+                cv2.putText(frame, "Capturing...", (30, 60), cv2.FONT_HERSHEY_TRIPLEX, 2, (127, 255, 255))
+                cv2.imwrite("gestures/" + str(g_id) + "/" + str(pic_no) + ".jpg", save_img)
+
+        cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
+        cv2.putText(frame, str(pic_no), (30, 400), cv2.FONT_HERSHEY_TRIPLEX, 1.5, (127, 127, 255))
+        cv2.imshow("Capturing gesture", frame)
+        cv2.imshow("thresh", thresh)
+        keypress = cv2.waitKey(1)
+        if keypress == ord('c'):
+            if flag_start_capturing == False:
+                flag_start_capturing = True
+            else:
+                flag_start_capturing = False
+                frames = 0
+        if flag_start_capturing == True:
+            frames += 1
+        if pic_no == total_pics:
+            break
+
+
+g_id = input("Enter gesture number: ")
+main(g_id)

examples/keras/Emojinator/Emojinator.py

@@ -0,0 +1,98 @@
+import cv2
+from keras.models import load_model
+import numpy as np
+import os
+
+model = load_model('emojinator.h5')
+
+def main():
+    emojis = get_emojis()
+    cap = cv2.VideoCapture(0)
+    x, y, w, h = 300, 50, 350, 350
+
+    while (cap.isOpened()):
+        ret, img = cap.read()
+        img = cv2.flip(img, 1)
+        hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
+        mask2 = cv2.inRange(hsv, np.array([2, 50, 60]), np.array([25, 150, 255]))
+        res = cv2.bitwise_and(img, img, mask=mask2)
+        gray = cv2.cvtColor(res, cv2.COLOR_BGR2GRAY)
+        median = cv2.GaussianBlur(gray, (5, 5), 0)
+
+        kernel_square = np.ones((5, 5), np.uint8)
+        dilation = cv2.dilate(median, kernel_square, iterations=2)
+        opening = cv2.morphologyEx(dilation, cv2.MORPH_CLOSE, kernel_square)
+        ret, thresh = cv2.threshold(opening, 30, 255, cv2.THRESH_BINARY)
+
+        thresh = thresh[y:y + h, x:x + w]
+        contours = cv2.findContours(thresh.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)[1]
+        if len(contours) > 0:
+            contour = max(contours, key=cv2.contourArea)
+            if cv2.contourArea(contour) > 2500:
+                x, y, w1, h1 = cv2.boundingRect(contour)
+                newImage = thresh[y:y + h1, x:x + w1]
+                newImage = cv2.resize(newImage, (50, 50))
+                pred_probab, pred_class = keras_predict(model, newImage)
+                print(pred_class, pred_probab)
+                img = overlay(img, emojis[pred_class], 400, 250, 90, 90)
+
+        x, y, w, h = 300, 50, 350, 350
+        cv2.imshow("Frame", img)
+        cv2.imshow("Contours", thresh)
+        k = cv2.waitKey(10)
+        if k == 27:
+            break
+
+def keras_predict(model, image):
+    processed = keras_process_image(image)
+    pred_probab = model.predict(processed)[0]
+    pred_class = list(pred_probab).index(max(pred_probab))
+    return max(pred_probab), pred_class
+
+
+def keras_process_image(img):
+    image_x = 50
+    image_y = 50
+    img = cv2.resize(img, (image_x, image_y))
+    img = np.array(img, dtype=np.float32)
+    img = np.reshape(img, (-1, image_x, image_y, 1))
+    return img
+
+def get_emojis():
+    emojis_folder = 'hand_emo/'
+    emojis = []
+    for emoji in range(len(os.listdir(emojis_folder))):
+        print(emoji)
+        emojis.append(cv2.imread(emojis_folder+str(emoji)+'.png', -1))
+    return emojis
+
+def overlay(image, emoji, x,y,w,h):
+    emoji = cv2.resize(emoji, (w, h))
+    try:
+        image[y:y+h, x:x+w] = blend_transparent(image[y:y+h, x:x+w], emoji)
+    except:
+        pass
+    return image
+
+def blend_transparent(face_img, overlay_t_img):
+    # Split out the transparency mask from the colour info
+    overlay_img = overlay_t_img[:,:,:3] # Grab the BRG planes
+    overlay_mask = overlay_t_img[:,:,3:]  # And the alpha plane
+
+    # Again calculate the inverse mask
+    background_mask = 255 - overlay_mask
+
+    # Turn the masks into three channel, so we can use them as weights
+    overlay_mask = cv2.cvtColor(overlay_mask, cv2.COLOR_GRAY2BGR)
+    background_mask = cv2.cvtColor(background_mask, cv2.COLOR_GRAY2BGR)
+
+    # Create a masked out face image, and masked out overlay
+    # We convert the images to floating point in range 0.0 - 1.0
+    face_part = (face_img * (1 / 255.0)) * (background_mask * (1 / 255.0))
+    overlay_part = (overlay_img * (1 / 255.0)) * (overlay_mask * (1 / 255.0))
+
+    # And finally just add them together, and rescale it back to an 8bit integer image
+    return np.uint8(cv2.addWeighted(face_part, 255.0, overlay_part, 255.0, 0.0))
+
+keras_predict(model, np.zeros((50, 50, 1), dtype=np.uint8))
+main()

examples/keras/Emojinator/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Akshay Bahadur
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

examples/keras/Emojinator/TrainEmojinator.py

@@ -0,0 +1,79 @@
+import numpy as np
+from keras import layers
+from keras.layers import Input, Dense, Activation, ZeroPadding2D, BatchNormalization, Flatten, Conv2D
+from keras.layers import AveragePooling2D, MaxPooling2D, Dropout, GlobalMaxPooling2D, GlobalAveragePooling2D
+from keras.utils import np_utils
+from keras.models import Sequential
+from keras.callbacks import ModelCheckpoint
+import pandas as pd
+
+import keras.backend as K
+
+def keras_model(image_x, image_y):
+    num_of_classes = 12
+    model = Sequential()
+    model.add(Conv2D(32, (5, 5), input_shape=(image_x, image_y, 1), activation='relu'))
+    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'))
+    model.add(Conv2D(64, (5, 5), activation='sigmoid'))
+    model.add(MaxPooling2D(pool_size=(5, 5), strides=(5, 5), padding='same'))
+    model.add(Flatten())
+    model.add(Dense(1024, activation='relu'))
+    model.add(Dropout(0.6))
+    model.add(Dense(num_of_classes, activation='softmax'))
+
+    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
+    filepath = "emojinator.h5"
+    checkpoint1 = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
+    callbacks_list = [checkpoint1]
+
+    return model, callbacks_list
+
+def main():
+    data = pd.read_csv("train_foo.csv")
+    dataset = np.array(data)
+    np.random.shuffle(dataset)
+    X = dataset
+    Y = dataset
+    X = X[:, 1:2501]
+    Y = Y[:, 0]
+
+    X_train = X[0:12000, :]
+    X_train = X_train / 255.
+    X_test = X[12000:13201, :]
+    X_test = X_test / 255.
+
+    # Reshape
+    Y = Y.reshape(Y.shape[0], 1)
+    Y_train = Y[0:12000, :]
+    Y_train = Y_train.T
+    Y_test = Y[12000:13201, :]
+    Y_test = Y_test.T
+
+    print("number of training examples = " + str(X_train.shape[0]))
+    print("number of test examples = " + str(X_test.shape[0]))
+    print("X_train shape: " + str(X_train.shape))
+    print("Y_train shape: " + str(Y_train.shape))
+    print("X_test shape: " + str(X_test.shape))
+    print("Y_test shape: " + str(Y_test.shape))
+    image_x = 50
+    image_y = 50
+
+    train_y = np_utils.to_categorical(Y_train)
+    test_y = np_utils.to_categorical(Y_test)
+    train_y = train_y.reshape(train_y.shape[1], train_y.shape[2])
+    test_y = test_y.reshape(test_y.shape[1], test_y.shape[2])
+    X_train = X_train.reshape(X_train.shape[0], 50, 50, 1)
+    X_test = X_test.reshape(X_test.shape[0], 50, 50, 1)
+    print("X_train shape: " + str(X_train.shape))
+    print("X_test shape: " + str(X_test.shape))
+
+    model, callbacks_list = keras_model(image_x, image_y)
+    model.fit(X_train, train_y, validation_data=(X_test, test_y), epochs=10, batch_size=64,
+              callbacks=callbacks_list)
+    scores = model.evaluate(X_test, test_y, verbose=0)
+    print("CNN Error: %.2f%%" % (100 - scores[1] * 100))
+
+    model.save('emojinator.h5')
+
+
+main()

examples/keras/Emojinator/readme.md

@@ -0,0 +1,49 @@
+# Emojinator   [![](https://img.shields.io/github/license/sourcerer-io/hall-of-fame.svg?colorB=ff0000)](https://github.com/akshaybahadur21/Emojinator/blob/master/LICENSE.md)  [![](https://img.shields.io/badge/Akshay-Bahadur-brightgreen.svg?colorB=ff0000)](https://akshaybahadur.com)
+
+This code helps you to recognize and classify different emojis. As of now, we are only supporting hand emojis. This is inspired by [Lobe.ai](https://lobe.ai/).
+
+# [Rock Paper Scissor Lizard Spock](https://github.com/akshaybahadur21/Emojinator/tree/master/Rock_Paper_Scissor_Lizard_Spock) [![](https://img.shields.io/github/license/sourcerer-io/hall-of-fame.svg?colorB=ff0000)](https://github.com/akshaybahadur21/Emojinator/blob/master/LICENSE.md)  [![](https://img.shields.io/badge/Akshay-Bahadur-brightgreen.svg?colorB=ff0000)](https://akshaybahadur.com)
+
+
+### Code Requirements
+You can install Conda for python which resolves all the dependencies for machine learning.
+
+##### pip install requirements.txt
+
+### Description
+Emojis are ideograms and smileys used in electronic messages and web pages. Emoji exist in various genres, including facial expressions, common objects, places and types of weather, and animals. They are much like emoticons, but emoji are actual pictures instead of typographics.
+
+### Functionalities
+1) Filters to detect hand.
+2) CNN for training the model.
+
+
+### Python  Implementation
+
+1) Network Used- Convolutional Neural Network
+
+If you face any problem, kindly raise an issue
+
+### Procedure
+
+1) First, you have to create a gesture database. For that, run `CreateGest.py`. Enter the gesture name and you will get 2 frames displayed. Look at the contour frame and adjust your hand to make sure that you capture the features of your hand. Press 'c' for capturing the images. It will take 1200 images of one gesture. Try moving your hand a little within the frame to make sure that your model doesn't overfit at the time of training.
+2) Repeat this for all the features you want.
+3) Run `CreateCSV.py` for converting the images to a CSV file
+4) If you want to train the model, run 'TrainEmojinator.py'
+5) Finally, run `Emojinator.py` for testing your model via webcam.
+
+### Contributors
+
+##### 1) [Akshay Bahadur](https://github.com/akshaybahadur21/)
+##### 2) [Raghav Patnecha](https://github.com/raghavpatnecha)
+
+### Emojinator
+<img src="https://github.com/akshaybahadur21/Emojinator/blob/master/emo.gif">
+
+### [Rock Paper Scissor Lizard Spock](https://github.com/akshaybahadur21/Emojinator/tree/master/Rock_Paper_Scissor_Lizard_Spock)
+<img src="https://github.com/akshaybahadur21/Emojinator/blob/master/RPS.gif">
+
+
+
+
+

examples/keras/Emojinator/requirements.txt

@@ -0,0 +1,7 @@
+numpy
+matplotlib
+cv2
+keras
+pandas
+h5py
+scipy

examples/keras/README.md

@@ -43,6 +43,10 @@ This is a Keras implementation of InfoGAN (Information-theoretic extension to th
 
 This is a Keras implementation of WGAN (Wasserstein Generative Adversarial Network), which can generates handwritten digits througth training.
 
+9. **Emojinator:**
+
+This is a Keras implementation inspired by Lobe.ai platform, which can classify different hand gestures into emojis through training.
+
 # How to Run
 
 1. Open the solution. (It will open with Visual Studio 2017 by default.)
@@ -71,4 +75,7 @@ These projects are contributed by University Students from Microsoft Student Clu
 3. Project DenseNet
 
     - Contributors: Secone Liu, Zou Ji, Yaxuan Dai, Jie Lin
-    - University: Beijing University of Post and Telecommunications
+    - University: Beijing University of Post and Telecommunications
+
+4. Emojinator
+    - Contributors: Akshay Bahadur, Raghav Patnecha