Video.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Video Input\n",
        "Computer Vision project Improvement/Feature.\n",
        "\n",
        "In the last segment of our main project code, we saved the model's weights locally for later use, so we can reuse the model later and without retraining it from scratch.\n",
        "Now, we would like to improve our project by adding a video classification feature using our trained model."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Define the model architecture"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "To load our model (basically the model weights), we need to define the models architecture and functionality, like a blueprint container our model can be load into. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {},
      "outputs": [],
      "source": [
        "import os\n",
        "import torch\n",
        "import torchvision\n",
        "import tarfile\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "from torchvision.datasets.utils import download_url\n",
        "from torch.utils.data import random_split\n",
        "from torch import nn\n",
        "import torch.nn.functional as F\n",
        "from torchvision.datasets import ImageFolder\n",
        "from torchvision.transforms import ToTensor"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {},
      "outputs": [],
      "source": [
        "def accuracy(outputs, labels):\n",
        "    _, preds = torch.max(outputs, dim=1)\n",
        "    return torch.tensor(torch.sum(preds == labels).item() / len(preds))\n",
        "\n",
        "class ImageClassificationBase(nn.Module):\n",
        "    def training_step(self, batch):\n",
        "        images, labels = batch \n",
        "        out = self(images)                  # Generate predictions\n",
        "        loss = F.cross_entropy(out, labels) # Calculate loss\n",
        "        return loss\n",
        "    \n",
        "    def validation_step(self, batch):\n",
        "        images, labels = batch \n",
        "        out = self(images)                    # Generate predictions\n",
        "        loss = F.cross_entropy(out, labels)   # Calculate loss\n",
        "        acc = accuracy(out, labels)           # Calculate accuracy\n",
        "        return {'val_loss': loss.detach(), 'val_acc': acc}\n",
        "        \n",
        "    def validation_epoch_end(self, outputs):\n",
        "        batch_losses = [x['val_loss'] for x in outputs]\n",
        "        epoch_loss = torch.stack(batch_losses).mean()   # Combine losses\n",
        "        batch_accs = [x['val_acc'] for x in outputs]\n",
        "        epoch_acc = torch.stack(batch_accs).mean()      # Combine accuracies\n",
        "        return {'val_loss': epoch_loss.item(), 'val_acc': epoch_acc.item()}\n",
        "    \n",
        "    def epoch_end(self, epoch, result):\n",
        "        print(\"Epoch [{}], train_loss: {:.4f}, val_loss: {:.4f}, val_acc: {:.4f}\".format(\n",
        "            epoch, result['train_loss'], result['val_loss'], result['val_acc']))\n",
        "\n",
        "class Fruits360CnnModel(ImageClassificationBase):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.network = nn.Sequential(\n",
        "            nn.Conv2d(3, 32, kernel_size=3, padding=1),\n",
        "            nn.ReLU(),\n",
        "            nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1),\n",
        "            nn.ReLU(),\n",
        "            nn.MaxPool2d(2, 2), # output: 64 x 50 x 50\n",
        "\n",
        "            nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),\n",
        "            nn.ReLU(),\n",
        "            nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),\n",
        "            nn.ReLU(),\n",
        "            nn.MaxPool2d(2, 2), # output: 128 x 25 x 25\n",
        "\n",
        "            nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1),\n",
        "            nn.ReLU(),\n",
        "            nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1),\n",
        "            nn.ReLU(),#output :256*25*25\n",
        "            nn.MaxPool2d(5, 5), # output: 256 x 5 x 5\n",
        "\n",
        "            nn.Flatten(), \n",
        "            nn.Linear(256*5*5, 1024),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(1024, 512),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(512, 131))\n",
        "            \n",
        "        \n",
        "    def forward(self, xb):\n",
        "        return self.network(xb)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [],
      "source": [
        "def get_default_device():\n",
        "    \"\"\"Pick GPU if available, else CPU\"\"\"\n",
        "    if torch.cuda.is_available():\n",
        "        return torch.device('cuda')\n",
        "    else:\n",
        "        return torch.device('cpu')\n",
        "    \n",
        "def to_device(data, device):\n",
        "    \"\"\"Move tensor(s) to chosen device\"\"\"\n",
        "    if isinstance(data, (list,tuple)):\n",
        "        return [to_device(x, device) for x in data]\n",
        "    return data.to(device, non_blocking=True)\n",
        "\n",
        "class DeviceDataLoader():\n",
        "    \"\"\"Wrap a dataloader to move data to a device\"\"\"\n",
        "    def __init__(self, dl, device):\n",
        "        self.dl = dl\n",
        "        self.device = device\n",
        "        \n",
        "    def __iter__(self):\n",
        "        \"\"\"Yield a batch of data after moving it to device\"\"\"\n",
        "        for b in self.dl: \n",
        "            yield to_device(b, self.device)\n",
        "\n",
        "    def __len__(self):\n",
        "        \"\"\"Number of batches\"\"\"\n",
        "        return len(self.dl)\n",
        "\n",
        "dataset = ImageFolder('fruits-360/Training', transform=ToTensor())\n",
        "\n",
        "def predict_image(img, model):\n",
        "    # Convert to a batch of 1\n",
        "    xb = to_device(img.unsqueeze(0), device)\n",
        "    # Get predictions from model\n",
        "    yb = model(xb)\n",
        "    # Pick index with highest probability\n",
        "    _, preds  = torch.max(yb, dim=1)\n",
        "    # Retrieve the class label\n",
        "    return dataset.classes[preds[0].item()]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [],
      "source": [
        "device = get_default_device()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Load the Model\n",
        "\n",
        "After defining the container for our model, we now can load it."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model = to_device(Fruits360CnnModel(), device)\n",
        "model.load_state_dict(torch.load('fruits360-cnn.pth'))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Video Detection\n",
        "\n",
        "Now that we have the model loaded, we can add the functionally of video classification. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {},
      "outputs": [],
      "source": [
        "import cv2 # (openCV) - provides a real-time optimized Computer Vision library, tools, and hardware.\n",
        "# import imutils\n",
        "import torchvision.transforms as transforms #allows transforming of numpy.arrays into tensors"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Defining the video capture object"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {},
      "outputs": [],
      "source": [
        "#vid = cv2.VideoCapture(0) # webcam\n",
        "vid = cv2.VideoCapture('fruits-demo.mp4') # video based on dataset\n",
        "#vid = cv2.VideoCapture('demo2.mp4') # video from youtube"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Perform video classification on the video capture object"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "while(vid.isOpened()):\n",
        "    # Capture the video frame by frame\n",
        "    ret, frame = vid.read()\n",
        "\n",
        "    # Display the resulting frame\n",
        "    cv2.imshow('frame', frame)\n",
        "\n",
        "    #resize the frame to 100x100\n",
        "    frame = cv2.resize(frame,(100,100))\n",
        "\n",
        "    # OpenCV uses BGR instead RGB By default, perform change to RGB\n",
        "    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n",
        "\n",
        "    #turn frame (numpy.array) to tensor\n",
        "    transform = transforms.ToTensor()\n",
        "    frame = transform(frame)\n",
        " \n",
        "    #use the model to predict class for the current frame\n",
        "    print('Predicted:', predict_image(frame, model))\n",
        "\n",
        "    # the 'q' button is set as the\n",
        "    # quitting button you may use any\n",
        "    # desired button of your choice\n",
        "    if cv2.waitKey(1) & 0xFF == ord('q'):\n",
        "        break\n",
        "  \n",
        "# After the loop release the cap object\n",
        "vid.release()\n",
        "# Destroy all the windows\n",
        "cv2.destroyAllWindows()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "TPU",
    "colab": {
      "collapsed_sections": [],
      "name": "computerVisionProject.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.7"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}