[Testing]_SIBI_Lang_Estimation.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cluqBPyMfRxv"
      },
      "outputs": [],
      "source": [
        "# import library and framework\n",
        "import cv2\n",
        "import csv\n",
        "import numpy as np\n",
        "import mediapipe as mp"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-Bn9k0isfUf1"
      },
      "outputs": [],
      "source": [
        "# create object for draw and get hand detection\n",
        "mp_drawing = mp.solutions.drawing_utils\n",
        "mp_drawing_styles = mp.solutions.drawing_styles\n",
        "mp_hands = mp.solutions.hands"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qy1xp9I5fH4l"
      },
      "outputs": [],
      "source": [
        "# Define global variable for hand keypoints data\n",
        "hand_keypoint_data = np.array([])\n",
        "\n",
        "# used to record the time when we processed last frame \n",
        "prev_frame_time = 0\n",
        "  \n",
        "# used to record the time at which we processed current frame \n",
        "new_frame_time = 0\n",
        "\n",
        "# Load Model\n",
        "with open(\"<insert your svm model>\", 'rb') as file:\n",
        "    action_model = pickle.load(file)\n",
        "\n",
        "# For webcam input:\n",
        "cap = cv2.VideoCapture(0)\n",
        "with mp_hands.Hands(\n",
        "    model_complexity=0,\n",
        "    min_detection_confidence=0.5,\n",
        "    min_tracking_confidence=0.5) as hands:\n",
        "  \n",
        "  # Read until video is completed\n",
        "  while cap.isOpened():\n",
        "    # Capture frame-by-frame\n",
        "    success, image = cap.read()\n",
        "    if not success:\n",
        "      print(\"Ignoring empty camera frame.\")\n",
        "      # If loading a video, use 'break' instead of 'continue'.\n",
        "      continue\n",
        "    \n",
        "    try:\n",
        "      # To improve performance, optionally mark the image as not writeable to\n",
        "      # pass by reference.\n",
        "      image.flags.writeable = False\n",
        "      image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
        "      results = hands.process(image)\n",
        "\n",
        "      # Draw the hand annotations on the image.\n",
        "      image.flags.writeable = True\n",
        "      image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n",
        "      if results.multi_hand_landmarks:\n",
        "        for hand_landmarks in results.multi_hand_landmarks:\n",
        "          mp_drawing.draw_landmarks(\n",
        "              image,\n",
        "              hand_landmarks,\n",
        "              mp_hands.HAND_CONNECTIONS,\n",
        "              mp_drawing_styles.get_default_hand_landmarks_style(),\n",
        "              mp_drawing_styles.get_default_hand_connections_style())\n",
        "      \n",
        "      # Checking keypoints if complete will do this block of code\n",
        "      if len(results.multi_hand_landmarks[0].landmark) >= 21:\n",
        "        # define variable for centering and scaling process\n",
        "        centering = np.array([])\n",
        "        scaling = np.array([])\n",
        "\n",
        "        # Centering X coordinate Process\n",
        "        for indexPoint in range(21):\n",
        "          centering = np.append(centering, (\n",
        "            results.multi_hand_landmarks[0].landmark[indexPoint].x - results.multi_hand_landmarks[0].landmark[0].x))\n",
        "\n",
        "        # Centering Y coordinate Process\n",
        "        for indexPoint in range(21):\n",
        "          centering = np.append(centering, (\n",
        "            results.multi_hand_landmarks[0].landmark[indexPoint].y - results.multi_hand_landmarks[0].landmark[0].y))\n",
        "\n",
        "        centering = centering.reshape(2, 21)\n",
        "        \n",
        "        # Scaling Process\n",
        "        for indexIter in range(2):\n",
        "          for jointIter in range(21):\n",
        "            scaling = np.append(scaling, centering[indexIter][jointIter] / np.max(\n",
        "              np.absolute(centering[indexIter])) * 320)\n",
        "        \n",
        "        # Normalization Process\n",
        "        for jointIter in range(42):\n",
        "          hand_keypoint_data = np.append(hand_keypoint_data, (scaling[jointIter] + 320))\n",
        "\n",
        "        # Write spatiodata from hand keypoints coordinate\n",
        "        if len(hand_keypoint_data) >= 210:\n",
        "          # predict the SIBI language\n",
        "          prediction = action_model.predict([hand_keypoint_data])\n",
        "\n",
        "          cv2.putText(image,f'{prediction[0]}', (10,50), cv2.FONT_HERSHEY_SIMPLEX, 2, (100, 255, 0), 3, cv2.LINE_AA)\n",
        "\n",
        "          # deleted 42 old data \n",
        "          deletedIndex = np.arange(42)\n",
        "          hand_keypoint_data = np.delete(hand_keypoint_data, deletedIndex)\n",
        "\n",
        "    except Exception as e:\n",
        "      continue\n",
        "\n",
        "    finally:\n",
        "      # font which we will be using to display FPS\n",
        "      font = cv2.FONT_HERSHEY_SIMPLEX\n",
        "\n",
        "      # time when we finish processing for this frame\n",
        "      new_frame_time = time.time()\n",
        "\n",
        "      fps = 1 / (new_frame_time - prev_frame_time)\n",
        "      prev_frame_time = new_frame_time\n",
        "\n",
        "      # converting the fps into integer\n",
        "      fps = int(round(fps))\n",
        "\n",
        "      # converting the fps to string so that we can display it on frame\n",
        "      # by using putText function\n",
        "      fps = str(fps)\n",
        "\n",
        "      # puting the FPS count on the frame\n",
        "      cv2.putText(image, fps, (550, 50), font, 2, (100, 255, 0), 3, cv2.LINE_AA)\n",
        "\n",
        "      # Show the result\n",
        "      cv2.imshow('Result', image)\n",
        "\n",
        "      if cv2.waitKey(5) & 0xFF == 27:\n",
        "        break\n",
        "\n",
        "cap.release()\n",
        "cv2.destroyAllWindows()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "[Testing] SIBI-Lang-Estimation.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3.7.11 ('my_env')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.7.11"
    },
    "vscode": {
      "interpreter": {
        "hash": "3fe420644837b9164684aa99b0054c85678c51b2b00629249570993b2a64553c"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}