Memotion.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.9"
    },
    "colab": {
      "name": "Github file",
      "provenance": [],
      "collapsed_sections": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "9YAP37fzwjR8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%tensorflow_version 1.x"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BdBnrycFmFqe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install ekphrasis"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sMVeH1yHep6A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import string\n",
        "from nltk.corpus import stopwords\n",
        "import re\n",
        "\n",
        "from ekphrasis.classes.preprocessor import TextPreProcessor\n",
        "from ekphrasis.classes.tokenizer import SocialTokenizer\n",
        "from ekphrasis.dicts.emoticons import emoticons"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-YgoIZNEep6I",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text_processor = TextPreProcessor(\n",
        "    # terms that will be normalized\n",
        "    normalize=['url', 'email', 'percent', 'money', 'phone', 'user',\n",
        "        'time', 'url', 'date', 'number'],\n",
        "    # terms that will be annotated\n",
        "    annotate={\"hashtag\", \"allcaps\", \"elongated\", \"repeated\",\n",
        "        'emphasis', 'censored'},\n",
        "    fix_html=True,  # fix HTML tokens\n",
        "    \n",
        "    # corpus from which the word statistics are going to be used \n",
        "    # for word segmentation \n",
        "    segmenter=\"twitter\", \n",
        "    \n",
        "    # corpus from which the word statistics are going to be used \n",
        "    # for spell correction\n",
        "    corrector=\"twitter\", \n",
        "    \n",
        "    unpack_hashtags=True,  # perform word segmentation on hashtags\n",
        "    unpack_contractions=True,  # Unpack contractions (can't -> can not)\n",
        "    spell_correct_elong=True,  # spell correction for elongated words\n",
        "    \n",
        "    # select a tokenizer. You can use SocialTokenizer, or pass your own\n",
        "    # the tokenizer, should take as input a string and return a list of tokens\n",
        "    tokenizer=SocialTokenizer(lowercase=True).tokenize,\n",
        "    \n",
        "    # list of dictionaries, for replacing tokens extracted from the text,\n",
        "    # with other expressions. You can pass more than one dictionaries.\n",
        "    dicts=[emoticons]\n",
        ")\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_PGia_3mep6O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def print_text(texts,i,j):\n",
        "    for u in range(i,j):\n",
        "        print(texts[u])\n",
        "        print()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t1f1Ewvgep6T",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.read_csv(\"/content/drive/My Drive/Memotion /memotion_dataset_7k/labels.csv\", header=None, skiprows=1)\n",
        "text_array = df[3]\n",
        "image_path = df[1]\n",
        "print(\"Length of training data:\",len(text_array))\n",
        "print(\"Total images:\",len(image_path))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XibiXL1VVwTQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_val = pd.read_csv(\"/content/drive/My Drive/semeval-2020_trialdata/data1.csv\", header=None, skiprows=1)\n",
        "text_array_val = df_val[2]\n",
        "image_path_val = df_val[0]\n",
        "print(\"Length of validation data:\",len(text_array_val))\n",
        "print(\"No. if memes in validation set:\",len(image_path_val))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "noS86xzB6XSx",
        "colab_type": "text"
      },
      "source": [
        "**Retrieving embedding vectors for the images:**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AOdg5oaSep6Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Loading image encodings\n",
        "def map_func(img_name):\n",
        "    encoding = np.load(\"/content/drive/My Drive/Memotion /resnet_train/\"+img_name +'.npy')\n",
        "    return np.asarray(encoding)\n",
        "\n",
        "image_encodings = np.zeros((len(image_path),2048))\n",
        "for index in range(0,len(image_path)):\n",
        "    if(index%100 == 0):\n",
        "        print(str(index)+\"th entry done....\")\n",
        "    image_encodings[index] = map_func(image_path[index])\n",
        "    \n",
        "print(\"Images after getting encoded from resnet:\")\n",
        "print(np.shape(image_encodings))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g6kJVZZAep6e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(\"Size of text array: \" + str(np.size(text_array)))\n",
        "print(\"Size of image embeddings: \" + str(np.shape(image_encodings)))\n",
        "index = 0\n",
        "print(\"Image name: \" + image_path[index])\n",
        "print(\"Image text: \" + text_array[index])\n",
        "print(\"Size of image encoding: \" + str(np.shape(image_encodings[index])))\n",
        "print(\"Image embedding: \" )\n",
        "print(image_encodings[index])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eSkf5zIyZ_gb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def map_func(img_name):\n",
        "    encoding = np.load(\"/content/drive/My Drive/Memotion /resnet_trial/\"+img_name +'.npy')\n",
        "    return np.asarray(encoding)\n",
        "\n",
        "image_encodings_val = np.zeros((len(image_path_val),2048))\n",
        "for index in range(0,len(image_path_val)):\n",
        "    if(index%100 == 0):\n",
        "        print(str(index)+\"th entry done....\")\n",
        "    image_encodings_val[index] = map_func(image_path_val[index])\n",
        "    \n",
        "print(\"Images after getting encoded from resnet:\")\n",
        "print(np.shape(image_encodings_val))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a2BQ6IWdjxg5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(\"Size of validation text array: \" + str(np.size(text_array_val)))\n",
        "print(\"Size of validation image embeddings: \" + str(np.shape(image_encodings_val)))\n",
        "index = 0\n",
        "print(\"Image name: \" + image_path_val[index])\n",
        "print(\"Image text: \" + text_array_val[index])\n",
        "print(\"Size of image encoding: \" + str(np.shape(image_encodings_val[index])))\n",
        "print(\"Image embedding: \" )\n",
        "print(image_encodings_val[index])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uyTfJ1hJZDw0",
        "colab_type": "text"
      },
      "source": [
        "<h3> Pre-processing </h3>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fGmDh8A2JJgE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Removing nan from train data\n",
        "# Eg. Index 29 contained a NaN value, we replace that with a none keyword\n",
        "flag = text_array.isnull()\n",
        "for i in range(0,len(text_array)):\n",
        "  if flag[i]:\n",
        "      text_array[i] = \"none\"\n",
        "print(text_array[4799])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PgZ73pxFrHIN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Removing nan from trial data\n",
        "# Eg. Index 29 contained a NaN value, we replace that with a none keyword\n",
        "flag_val = text_array_val.isnull()\n",
        "for i in range(0,len(text_array_val)):\n",
        "  if flag_val[i]:\n",
        "      text_array_val[i] = \"none\"\n",
        "print(text_array_val[29])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rqLf8Vrnep6l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#removing website names\n",
        "def remove_website(text):\n",
        "    return \" \".join([word if re.search(\"r'https?://\\S+|www\\.\\S+'|((?i).com$|.co|.net)\",word)==None else \"\" for word in text.split(\" \") ])\n",
        "\n",
        "# Training set \n",
        "text_array = text_array.apply(lambda text: remove_website(text))\n",
        "print_text(text_array,99,101)\n",
        "\n",
        "print(\"#########################################################################\")\n",
        "\n",
        "# Validation set\n",
        "text_array_val = text_array_val.apply(lambda text: remove_website(text))\n",
        "print_text(text_array_val,99,101)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VLM4hpT5ep6r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Functions for chat word conversion\n",
        "f = open(\"/content/drive/My Drive/Memotion /slang.txt\", \"r\")\n",
        "chat_words_str = f.read()\n",
        "chat_words_map_dict = {}\n",
        "chat_words_list = []\n",
        "\n",
        "for line in chat_words_str.split(\"\\n\"):\n",
        "    if line != \"\":\n",
        "        cw = line.split(\"=\")[0]\n",
        "        cw_expanded = line.split(\"=\")[1]\n",
        "        chat_words_list.append(cw)\n",
        "        chat_words_map_dict[cw] = cw_expanded\n",
        "chat_words_list = set(chat_words_list)\n",
        "\n",
        "def chat_words_conversion(text):\n",
        "    new_text = []\n",
        "    for w in text.split():\n",
        "        if w.upper() in chat_words_list:\n",
        "            new_text.append(chat_words_map_dict[w.upper()])\n",
        "        else:\n",
        "            new_text.append(w)\n",
        "    return \" \".join(new_text)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_EettYCGep6w",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Chat word conversion\n",
        "# Training set\n",
        "text_array = text_array.apply(lambda text: chat_words_conversion(text))\n",
        "print_text(text_array,98,100)\n",
        "\n",
        "print(\"#########################################################################\")\n",
        "\n",
        "# Validation set\n",
        "text_array_val = text_array_val.apply(lambda text: chat_words_conversion(text))\n",
        "print_text(text_array_val,98,100)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8jpkT6aDep60",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Function for emoticon conversion\n",
        "from emoticons import EMOTICONS\n",
        "\n",
        "def convert_emoticons(text):\n",
        "    for emot in EMOTICONS:\n",
        "        text = re.sub(u'('+emot+')', \" \".join(EMOTICONS[emot].replace(\",\",\"\").split()), text)\n",
        "    return text\n",
        "\n",
        "\n",
        "#testing the emoticon function\n",
        "text = \"Hello :-) :-)\"\n",
        "text = convert_emoticons(text)\n",
        "print(text + \"\\n\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3AVC-DW189AM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Emoticon conversion\n",
        "\n",
        "# Training set\n",
        "text_array = text_array.apply(lambda text: convert_emoticons(text))\n",
        "print_text(text_array,0,5)\n",
        "\n",
        "print(\"##########################################################################\")\n",
        "\n",
        "# Validation set\n",
        "text_array_val = text_array_val.apply(lambda text: convert_emoticons(text))\n",
        "print_text(text_array_val,0,5)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YKR-9nrmep66",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Ekphrasis pipe for text pre-processing\n",
        "def ekphrasis_pipe(sentence):\n",
        "    cleaned_sentence = \" \".join(text_processor.pre_process_doc(sentence))\n",
        "    return cleaned_sentence\n",
        "\n",
        "# Training set\n",
        "text_array = text_array.apply(lambda text: ekphrasis_pipe(text))\n",
        "# Validation set\n",
        "text_array_val = text_array_val.apply(lambda text: ekphrasis_pipe(text))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EzHCGCyBep6-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Testing validation data after ekphrasis pipe\n",
        "print_text(text_array_val,110,115)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BUnVWgoSep7D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Removing unnecessary punctuations\n",
        "PUNCT_TO_REMOVE = \"\\\"$%&'()+,-./;=[\\]^_`{|}~\"\n",
        "def remove_punctuation(text):\n",
        "    return text.translate(str.maketrans('', '', PUNCT_TO_REMOVE))\n",
        "\n",
        "# Training set\n",
        "text_array = text_array.apply(lambda text: remove_punctuation(text))\n",
        "# Validation set\n",
        "text_array_val = text_array_val.apply(lambda text: remove_punctuation(text))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TUAMPQUOep7J",
        "colab_type": "text"
      },
      "source": [
        "<h2>Text Pre processing complete</h2>\n",
        "<p>Data stored in text_array</p>\n",
        "<p>Validation data in text_array_val</p>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MwikFuJ5ep7L",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Finding length of longest array\n",
        "maxLen = len(max(text_array,key = lambda text: len(text.split(\" \"))).split(\" \"))\n",
        "print(maxLen)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P6pQDHf8ep7P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Printing the length longest N sentences  \n",
        "u = lambda text: len(text.split(\" \"))\n",
        "sentence_lengths = []\n",
        "for x in text_array:\n",
        "    sentence_lengths.append(u(x))\n",
        "print(sorted(sentence_lengths)[-70:])\n",
        "print(len(sentence_lengths))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oSmEIUgjep7V",
        "colab_type": "text"
      },
      "source": [
        "<h2>Reading glove values</h2>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j3LwVe2zep7W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def read_glove_vecs(glove_file):\n",
        "    with open(glove_file, 'r', encoding=\"utf8\") as f:\n",
        "        words = set()\n",
        "        word_to_vec_map = {}\n",
        "        for line in f:\n",
        "            line = line.strip().split()\n",
        "            curr_word = line[0]\n",
        "            words.add(curr_word)\n",
        "            word_to_vec_map[curr_word] = np.array(line[1:], dtype=np.float64)\n",
        "        \n",
        "        i = 1\n",
        "        words_to_index = {}\n",
        "        index_to_words = {}\n",
        "        for w in sorted(words):\n",
        "            words_to_index[w] = i\n",
        "            index_to_words[i] = w\n",
        "            i = i + 1\n",
        "    return words_to_index, index_to_words, word_to_vec_map, words"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ndHwAqpRep7b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We need word_to_index and index_to_word for creating the embedding layer.\n",
        "word_to_index, index_to_word, word_to_vec_map, vocab = read_glove_vecs('/content/drive/My Drive/Memotion /glove.twitter.27B.100d.txt')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZYcWXXqnw8Wh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Function for rectifying the error in column values. Some rows were shifted to left by 1 position\n",
        "for i in range(0,len(df_val[8])):\n",
        "  if df_val[8][i] == \"image_and_text \" or df_val[8][i] == \"image\" or df_val[8][i] == \"text\":\n",
        "    df_val[8][i] = df_val[7][i]\n",
        "    df_val[7][i] = df_val[6][i]\n",
        "    df_val[6][i] = df_val[5][i]\n",
        "    df_val[5][i] = df_val[4][i]\n",
        "    df_val[4][i] = df_val[3][i]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JnKjeM0tep7o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Count of each label in dataset\n",
        "from collections import Counter\n",
        "\n",
        "# Training set states for Subtask A\n",
        "state = np.asarray(df[8])\n",
        "state_val = np.asarray(df_val[8])\n",
        "\n",
        "# Training set states for humour\n",
        "state_humour = np.asarray(df[4])\n",
        "state_humour_val = np.asarray(df_val[4])\n",
        "\n",
        "# Training set states for sarcasm\n",
        "state_sarcasm = np.asarray(df[5])\n",
        "state_sarcasm_val = np.asarray(df_val[5])\n",
        "\n",
        "# Training set states for offensive\n",
        "state_offense = np.asarray(df[6])\n",
        "state_offense_val = np.asarray(df_val[6])\n",
        "\n",
        "# Training set states for motivation\n",
        "state_motivate = np.asarray(df[7])\n",
        "state_motivate_val = np.asarray(df_val[7])\n",
        "\n",
        "# Printing counts for analysis\n",
        "# Normal states\n",
        "print(\"Elements: \",set(state))\n",
        "print(\"Length: \",len(state))\n",
        "print(Counter(state))\n",
        "print(\"##########################################################################\")\n",
        "print(\"Elements: \",set(state_val))\n",
        "print(\"Length: \",len(state_val))\n",
        "print(Counter(state_val))\n",
        "\n",
        "# Humour states\n",
        "print(\"\\n****************************************************************************************************\\n\")\n",
        "print(\"Elements: \",set(state_humour))\n",
        "print(\"Length: \",len(state_humour))\n",
        "print(Counter(state_humour))\n",
        "print(\"##########################################################################\")\n",
        "print(\"Elements: \",set(state_humour_val))\n",
        "print(\"Length: \",len(state_humour_val))\n",
        "print(Counter(state_humour_val))\n",
        "\n",
        "# Sarcasm states\n",
        "print(\"\\n****************************************************************************************************\\n\")\n",
        "print(\"Elements: \",set(state_sarcasm))\n",
        "print(\"Length: \",len(state_sarcasm))\n",
        "print(Counter(state_sarcasm))\n",
        "print(\"##########################################################################\")\n",
        "print(\"Elements: \",set(state_sarcasm_val))\n",
        "print(\"Length: \",len(state_sarcasm_val))\n",
        "print(Counter(state_sarcasm_val))\n",
        "\n",
        "# Offense states\n",
        "print(\"\\n****************************************************************************************************\\n\")\n",
        "print(\"Elements: \",set(state_offense))\n",
        "print(\"Length: \",len(state_offense))\n",
        "print(Counter(state_offense))\n",
        "print(\"##########################################################################\")\n",
        "print(\"Elements: \",set(state_offense_val))\n",
        "print(\"Length: \",len(state_offense_val))\n",
        "print(Counter(state_offense_val))\n",
        "\n",
        "# Motivate states\n",
        "print(\"\\n****************************************************************************************************\\n\")\n",
        "print(\"Elements: \",set(state_motivate))\n",
        "print(\"Length: \",len(state_motivate))\n",
        "print(Counter(state_motivate))\n",
        "print(\"##########################################################################\")\n",
        "print(\"Elements: \",set(state_motivate_val))\n",
        "print(\"Length: \",len(state_motivate_val))\n",
        "print(Counter(state_motivate_val))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OHWLf8sPep7t",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Subtask A\n",
        "# Converting the Labels to integer values for conversion to one hot vectors\n",
        "Y = []\n",
        "Y_val = []\n",
        "\n",
        "Yn = []\n",
        "Yn_val = []\n",
        "\n",
        "# Training set    \n",
        "for i in range(0,len(state)):\n",
        "    if(state[i] == \"neutral\"):\n",
        "        Y.append(0)\n",
        "        Yn.append(0)\n",
        "    if(state[i] == \"positive\"):\n",
        "        Y.append(1)\n",
        "        Yn.append(1)\n",
        "    if(state[i] == \"very_positive\"):\n",
        "        Y.append(1)\n",
        "        Yn.append(2)\n",
        "    if(state[i] == \"negative\"):\n",
        "        Y.append(2)\n",
        "        Yn.append(3)\n",
        "    if(state[i] == \"very_negative\"):\n",
        "        Y.append(2)\n",
        "        Yn.append(4)\n",
        "\n",
        "# Validation set\n",
        "for i in range(0,914):\n",
        "    if(state_val[i] == \"neutral\"):\n",
        "        Y_val.append(0)\n",
        "        Yn_val.append(0)\n",
        "    if(state_val[i] == \"positive\"):\n",
        "        Y_val.append(1)\n",
        "        Yn_val.append(1)\n",
        "    if(state_val[i] == \"very_positive\"):\n",
        "        Y_val.append(1)\n",
        "        Yn_val.append(2)\n",
        "    if(state_val[i] == \"negative\"):\n",
        "        Y_val.append(2)\n",
        "        Yn_val.append(3)\n",
        "    if(state_val[i] == \"very_negative\"):\n",
        "        Y_val.append(2)\n",
        "        Yn_val.append(4)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BwKMECbr-Tmu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(Counter(state))\n",
        "print(Counter(Y),len(Y))\n",
        "print(Counter(Yn),len(Yn))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f_H50jnm-Jxf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Subtask B and Subtask C ---------- Humour\n",
        "# Converting the Labels to integer values for conversion to one hot vectors\n",
        "Y_humour = []\n",
        "Y_humour_val = []\n",
        "\n",
        "Yn_humour = []\n",
        "Yn_humour_val = []\n",
        "\n",
        "Ynf_humour = []\n",
        "Ynf_humour_val = []\n",
        "\n",
        "# 'not_funny', 'hilarious', 'funny', 'very_funny'\n",
        "# Training set    \n",
        "for i in range(0,len(state)):\n",
        "    if(state_humour[i] == \"not_funny\"):\n",
        "        Y_humour.append(0)\n",
        "        Yn_humour.append(0)\n",
        "        Ynf_humour.append(0)\n",
        "    if(state_humour[i] == \"funny\"):\n",
        "        Y_humour.append(1)\n",
        "        Yn_humour.append(1)\n",
        "        Ynf_humour.append(1)\n",
        "    if(state_humour[i] == \"very_funny\" or state_humour[i] == \"hilarious\"):\n",
        "        Y_humour.append(2)\n",
        "        Ynf_humour.append(1)\n",
        "        if(state_humour[i] == \"very_funny\"):\n",
        "          Yn_humour.append(2)\n",
        "        else:\n",
        "          Yn_humour.append(3)\n",
        "\n",
        "# Validation set\n",
        "for i in range(0,914):\n",
        "    if(state_humour_val[i] == \"not_funny\"):\n",
        "        Y_humour_val.append(0)\n",
        "        Yn_humour_val.append(0)\n",
        "        Ynf_humour_val.append(0)\n",
        "    if(state_humour_val[i] == \"funny\"):\n",
        "        Y_humour_val.append(1)\n",
        "        Yn_humour_val.append(1)\n",
        "        Ynf_humour_val.append(1)\n",
        "    if(state_humour_val[i] == \"very_funny\" or state_humour_val[i] == \"hilarious\"):\n",
        "        Y_humour_val.append(2)\n",
        "        Ynf_humour_val.append(1)\n",
        "        if(state_humour_val[i] == \"very_funny\"):\n",
        "          Yn_humour_val.append(2)\n",
        "        else:\n",
        "          Yn_humour_val.append(3)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nB5_P_ShBsJH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(Counter(state_humour))\n",
        "print(Counter(Y_humour),len(Y_humour))\n",
        "print(Counter(Yn_humour),len(Yn_humour))\n",
        "print(Counter(Ynf_humour),len(Ynf_humour))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wCwImJ5HCDdd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Subtask B and Subtask C ---------- Sarcasm\n",
        "# Converting the Labels to integer values for conversion to one hot vectors\n",
        "Y_sarcasm = []\n",
        "Y_sarcasm_val = []\n",
        "\n",
        "Yn_sarcasm = []\n",
        "Yn_sarcasm_val = []\n",
        "\n",
        "Ynf_sarcasm = []\n",
        "Ynf_sarcasm_val = []\n",
        "\n",
        "# 'general', 'not_sarcastic', 'twisted_meaning', 'very_twisted'\n",
        "# Training set    \n",
        "for i in range(0,len(state)):\n",
        "    if(state_sarcasm[i] == \"not_sarcastic\"):\n",
        "        Y_sarcasm.append(0)\n",
        "        Yn_sarcasm.append(0)\n",
        "        Ynf_sarcasm.append(0)\n",
        "    if(state_sarcasm[i] == \"general\"):\n",
        "        Y_sarcasm.append(1)\n",
        "        Yn_sarcasm.append(1)\n",
        "        Ynf_sarcasm.append(1)\n",
        "    if(state_sarcasm[i] == \"very_twisted\" or state_sarcasm[i] == \"twisted_meaning\"):\n",
        "        Y_sarcasm.append(2)\n",
        "        Ynf_sarcasm.append(1)\n",
        "        if(state_sarcasm[i] == \"twisted_meaning\"):\n",
        "          Yn_sarcasm.append(2)\n",
        "        else:\n",
        "          Yn_sarcasm.append(3)\n",
        "\n",
        "# Validation set\n",
        "for i in range(0,914):\n",
        "    if(state_sarcasm_val[i] == \"not_sarcastic\"):\n",
        "        Y_sarcasm_val.append(0)\n",
        "        Yn_sarcasm_val.append(0)\n",
        "        Ynf_sarcasm_val.append(0)\n",
        "    if(state_sarcasm_val[i] == \"general\"):\n",
        "        Y_sarcasm_val.append(1)\n",
        "        Yn_sarcasm_val.append(1)\n",
        "        Ynf_sarcasm_val.append(1)\n",
        "    if(state_sarcasm_val[i] == \"very_twisted\" or state_sarcasm_val[i] == \"twisted_meaning\"):\n",
        "        Y_sarcasm_val.append(2)\n",
        "        Ynf_sarcasm_val.append(1)\n",
        "        if(state_sarcasm_val[i] == \"twisted_meaning\"):\n",
        "          Yn_sarcasm_val.append(2)\n",
        "        else:\n",
        "          Yn_sarcasm_val.append(3)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2ROyyk8xCFm1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(Counter(state_sarcasm))\n",
        "print(Counter(Y_sarcasm),len(Y_sarcasm))\n",
        "print(Counter(Yn_sarcasm),len(Yn_sarcasm))\n",
        "print(Counter(Ynf_sarcasm),len(Ynf_sarcasm))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dDNWgT8BFRRQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Subtask B and Subtask C ---------- Offense\n",
        "# Converting the Labels to integer values for conversion to one hot vectors\n",
        "Y_offense = []\n",
        "Y_offense_val = []\n",
        "\n",
        "Yn_offense = []\n",
        "Yn_offense_val = []\n",
        "\n",
        "Ynf_offense = []\n",
        "Ynf_offense_val = []\n",
        "\n",
        "# 'very_offensive', 'not_offensive', 'slight', 'hateful_offensive'\n",
        "# Training set    \n",
        "for i in range(0,len(state)):\n",
        "    if(state_offense[i] == \"not_offensive\"):\n",
        "        Y_offense.append(0)\n",
        "        Yn_offense.append(0)\n",
        "        Ynf_offense.append(0)\n",
        "    if(state_offense[i] == \"slight\"):\n",
        "        Y_offense.append(1)\n",
        "        Yn_offense.append(1)\n",
        "        Ynf_offense.append(1)\n",
        "    if(state_offense[i] == \"very_offensive\" or state_offense[i] == \"hateful_offensive\"):\n",
        "        Y_offense.append(2)\n",
        "        Ynf_offense.append(1)\n",
        "        if(state_offense[i] == \"very_offensive\"):\n",
        "          Yn_offense.append(2)\n",
        "        else:\n",
        "          Yn_offense.append(3)\n",
        "\n",
        "# Validation set\n",
        "for i in range(0,914):\n",
        "    if(state_offense_val[i] == \"not_offensive\"):\n",
        "        Y_offense_val.append(0)\n",
        "        Yn_offense_val.append(0)\n",
        "        Ynf_offense_val.append(0)\n",
        "    if(state_offense_val[i] == \"slight\"):\n",
        "        Y_offense_val.append(1)\n",
        "        Yn_offense_val.append(1)\n",
        "        Ynf_offense_val.append(1)\n",
        "    if(state_offense_val[i] == \"very_offensive\" or state_offense_val[i] == \"hateful_offensive\"):\n",
        "        Y_offense_val.append(2)\n",
        "        Ynf_offense_val.append(1)\n",
        "        if(state_offense_val[i] == \"very_offensive\"):\n",
        "          Yn_offense_val.append(2)\n",
        "        else:\n",
        "          Yn_offense_val.append(3)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w1dGuWxyFRxo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(Counter(state_offense))\n",
        "print(Counter(Y_offense),len(Y_offense))\n",
        "print(Counter(Yn_offense),len(Yn_offense))\n",
        "print(Counter(Ynf_offense),len(Ynf_offense))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gO2pkFSpHYhF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Subtask B and Subtask C ---------- Motivate\n",
        "# Converting the Labels to integer values for conversion to one hot vectors\n",
        "Y_motivate = []\n",
        "Y_motivate_val = []\n",
        "\n",
        "# 'not_motivational', 'motivational'\n",
        "# Training set    \n",
        "for i in range(0,len(state)):\n",
        "    if(state_motivate[i] == \"motivational\"):\n",
        "      Y_motivate.append(1)\n",
        "    else:\n",
        "      Y_motivate.append(0)\n",
        "\n",
        "# Validation set\n",
        "for i in range(0,914):\n",
        "    if(state_motivate_val[i] == \"motivational\"):\n",
        "      Y_motivate_val.append(1)\n",
        "    else:\n",
        "      Y_motivate_val.append(0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ov4GxoYlHZBh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(Counter(state_motivate))\n",
        "print(Counter(Y_motivate),len(Y_motivate))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_TYBek-f9W2A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Testing the conversion into integers\n",
        "for i in range(0,5):\n",
        "  print(text_array_val[i])\n",
        "  print(state_val[i],Yn_val[i])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LRzfq_Erep7y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Verifying train set \n",
        "X = np.asarray(list(text_array))\n",
        "\n",
        "# Subtask A\n",
        "Y = np.asarray(list(Y))\n",
        "Yn = np.asarray(list(Yn))\n",
        "\n",
        "# Subtask B,C --- Humour\n",
        "Y_humour = np.asarray(list(Y_humour))\n",
        "Yn_humour = np.asarray(list(Yn_humour))\n",
        "Ynf_humour = np.asarray(list(Ynf_humour))\n",
        "\n",
        "# Subtask B,C --- Sarcasm\n",
        "Y_sarcasm = np.asarray(list(Y_sarcasm))\n",
        "Yn_sarcasm = np.asarray(list(Yn_sarcasm))\n",
        "Ynf_sarcasm = np.asarray(list(Ynf_sarcasm))\n",
        "\n",
        "# Subtask B,C --- Offense\n",
        "Y_offense = np.asarray(list(Y_offense))\n",
        "Yn_offense = np.asarray(list(Yn_offense))\n",
        "Ynf_offense = np.asarray(list(Ynf_offense))\n",
        "\n",
        "# Subtask B,C --- Motivation\n",
        "Y_motivate = np.asarray(list(Y_motivate))\n",
        "\n",
        "print(type(X))\n",
        "print(type(image_encodings))\n",
        "print(np.shape(X),np.shape(Y),np.shape(image_encodings))\n",
        "\n",
        "# Verifying validation set ------------------------------\n",
        "X_val = np.asarray(list(text_array_val))\n",
        "\n",
        "# Subtask A\n",
        "Y_val = np.asarray(list(Y_val))\n",
        "Yn_val = np.asarray(list(Yn_val))\n",
        "\n",
        "# Subtask B,C --- Humour\n",
        "Y_humour_val = np.asarray(list(Y_humour_val))\n",
        "Yn_humour_val = np.asarray(list(Yn_humour_val))\n",
        "Ynf_humour_val = np.asarray(list(Ynf_humour_val))\n",
        "\n",
        "# Subtask B,C --- Sarcasm\n",
        "Y_sarcasm_val = np.asarray(list(Y_sarcasm_val))\n",
        "Yn_sarcasm_val = np.asarray(list(Yn_sarcasm_val))\n",
        "Ynf_sarcasm_val = np.asarray(list(Ynf_sarcasm_val))\n",
        "\n",
        "# Subtask B,C --- Offense\n",
        "Y_offense_val = np.asarray(list(Y_offense_val))\n",
        "Yn_offense_val = np.asarray(list(Yn_offense_val))\n",
        "Ynf_offense_val = np.asarray(list(Ynf_offense_val))\n",
        "\n",
        "# Subtask B,C --- Motivation\n",
        "Y_motivate_val = np.asarray(list(Y_motivate_val))\n",
        "\n",
        "print(type(X_val))\n",
        "print(type(image_encodings_val))\n",
        "print(np.shape(X_val),np.shape(Y_val),np.shape(image_encodings_val))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uVOZCUtvep72",
        "colab_type": "text"
      },
      "source": [
        "<h3>Shuffling training and validation data</h3>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BqF3IZMJep73",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.utils import shuffle\n",
        "from sklearn.model_selection import train_test_split"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CeTRf81EKZ92",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X, image_encodings, state, state_humour, state_sarcasm, state_offense, state_motivate, Y, Yn, Y_humour, Yn_humour, Ynf_humour, Y_sarcasm, Yn_sarcasm, Ynf_sarcasm, Y_offense, Yn_offense, Ynf_offense, Y_motivate = shuffle(X, image_encodings, state, state_humour, state_sarcasm, state_offense, state_motivate, Y, Yn, Y_humour, Yn_humour, Ynf_humour, Y_sarcasm, Yn_sarcasm, Ynf_sarcasm, Y_offense, Yn_offense, Ynf_offense, Y_motivate, random_state=0)\n",
        "X, image_encodings, state, state_humour, state_sarcasm, state_offense, state_motivate, Y, Yn, Y_humour, Yn_humour, Ynf_humour, Y_sarcasm, Yn_sarcasm, Ynf_sarcasm, Y_offense, Yn_offense, Ynf_offense, Y_motivate = shuffle(X, image_encodings, state, state_humour, state_sarcasm, state_offense, state_motivate, Y, Yn, Y_humour, Yn_humour, Ynf_humour, Y_sarcasm, Yn_sarcasm, Ynf_sarcasm, Y_offense, Yn_offense, Ynf_offense, Y_motivate, random_state=1)\n",
        "X, image_encodings, state, state_humour, state_sarcasm, state_offense, state_motivate, Y, Yn, Y_humour, Yn_humour, Ynf_humour, Y_sarcasm, Yn_sarcasm, Ynf_sarcasm, Y_offense, Yn_offense, Ynf_offense, Y_motivate = shuffle(X, image_encodings, state, state_humour, state_sarcasm, state_offense, state_motivate, Y, Yn, Y_humour, Yn_humour, Ynf_humour, Y_sarcasm, Yn_sarcasm, Ynf_sarcasm, Y_offense, Yn_offense, Ynf_offense, Y_motivate, random_state=44)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lZx62-PA-tY9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Testing after Shuffling\n",
        "for i in range(40,50):\n",
        "  print(X[i])\n",
        "  print(state_humour[i],\":\",Y_humour[i],Yn_humour[i],Ynf_humour[i])\n",
        "print(Counter(state_humour))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dMtvLXcfep8O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(type(X))\n",
        "print(type(Y))\n",
        "print(np.shape(X),np.shape(Y),np.shape(image_encodings))\n",
        "print(np.shape(X_val),np.shape(Y_val),np.shape(image_encodings_val))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nU_7TDITep8Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Converting to one hot vectors\n",
        "def convert_to_one_hot(Y, C):\n",
        "    Y = np.eye(C)[Y.reshape(-1)] #u[Y] helps to index each element of Y index at u. U here is a class array\n",
        "    return Y"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IsNFFkpvep8j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Subtask A\n",
        "Y_oh_train = convert_to_one_hot(np.array(Y), C = 3)\n",
        "Y_oh_val = convert_to_one_hot(np.array(Y_val), C = 3)\n",
        "Y_oh_n_train = convert_to_one_hot(np.array(Yn), C = 5)\n",
        "Y_oh_n_val = convert_to_one_hot(np.array(Yn_val), C = 5)\n",
        "\n",
        "# Subtask B,C ---  Humour\n",
        "Y_oh_humour_train = convert_to_one_hot(np.array(Y_humour), C = 3)\n",
        "Y_oh_humour_val = convert_to_one_hot(np.array(Y_humour_val), C = 3)\n",
        "Y_oh_n_humour_train = convert_to_one_hot(np.array(Yn_humour), C = 4)\n",
        "Y_oh_n_humour_val = convert_to_one_hot(np.array(Yn_humour_val), C = 4)\n",
        "\n",
        "# Subtask B,C ---  Sarcasm\n",
        "Y_oh_sarcasm_train = convert_to_one_hot(np.array(Y_sarcasm), C = 3)\n",
        "Y_oh_sarcasm_val = convert_to_one_hot(np.array(Y_sarcasm_val), C = 3)\n",
        "Y_oh_n_sarcasm_train = convert_to_one_hot(np.array(Yn_sarcasm), C = 4)\n",
        "Y_oh_n_sarcasm_val = convert_to_one_hot(np.array(Yn_sarcasm_val), C = 4)\n",
        "\n",
        "# Subtask B,C ---  Offense\n",
        "Y_oh_offense_train = convert_to_one_hot(np.array(Y_offense), C = 3)\n",
        "Y_oh_offense_val = convert_to_one_hot(np.array(Y_offense_val), C = 3)\n",
        "Y_oh_n_offense_train = convert_to_one_hot(np.array(Yn_offense), C = 4)\n",
        "Y_oh_n_offense_val = convert_to_one_hot(np.array(Yn_offense_val), C = 4)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ye0ZDRm_d6m7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Testing conversion to One hot Vectors\n",
        "index = 861\n",
        "for i in range(0,6596):\n",
        "  if(state_offense[i] == 'hateful_offensive'):\n",
        "    index = i\n",
        "    break\n",
        "\n",
        "print(np.shape(Y_oh_offense_train))\n",
        "print(state_offense[index], Y_offense[index], \"is converted into one hot\", Y_oh_offense_train[index])\n",
        "\n",
        "print(np.shape(Y_oh_n_offense_train))\n",
        "print(state_offense[index], Yn_offense[index], \"is converted into one hot\", Y_oh_n_offense_train[index])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rblcEvi4ep8u",
        "colab_type": "text"
      },
      "source": [
        "<h2> Model using LSTM</h2>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2oxd7hi0ep8x",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "import keras\n",
        "import keras.backend as K\n",
        "from keras.layers import LeakyReLU\n",
        "from keras.models import Model\n",
        "from keras.layers import Dense, Input, Dropout, LSTM, Activation, Concatenate, Bidirectional, Add, RepeatVector, Dot, BatchNormalization, Flatten\n",
        "from keras.layers.embeddings import Embedding\n",
        "from keras.preprocessing import sequence\n",
        "from keras.initializers import glorot_uniform\n",
        "from keras.utils import plot_model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GQ7ZQM8R5NJ5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(tf.__version__)\n",
        "# K.clear_session()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yxfu7qzcep88",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Function to convert sentence to indices using the glove vocab\n",
        "def sentences_to_indices(X, word_to_index, max_len):\n",
        "    \"\"\"\n",
        "    Converts an array of sentences (strings) into an array of indices corresponding to words in the sentences.\n",
        "    \n",
        "    Arguments:\n",
        "    X -- array of sentences (strings)\n",
        "    word_to_index -- a dictionary containing the each word mapped to its index\n",
        "    max_len -- maximum number of words in a sentence. Sentences longer than this are truncated. \n",
        "    \n",
        "    Returns:\n",
        "    X_indices -- array of indices corresponding to words in the sentences from X\n",
        "    \"\"\"\n",
        "    \n",
        "    m = X.shape[0]                                   # number of training examples\n",
        "    X_indices = np.zeros((m,max_len))\n",
        "    \n",
        "    for i in range(m):                                \n",
        "        # Convert the ith training sentence in lower case and split is into words. You should get a list of words.\n",
        "        sentence_words =(X[i].lower()).split(\" \")\n",
        "        \n",
        "        j = 0        \n",
        "        for w in sentence_words:\n",
        "            if j==max_len:\n",
        "                break\n",
        "            # Set the (i,j)th entry of X_indices to the index of the correct word.\n",
        "            if w in word_to_index:\n",
        "                X_indices[i, j] = word_to_index[w]\n",
        "                # Increment j to j + 1\n",
        "                j = j+1\n",
        "  \n",
        "    return X_indices"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mdhzRvfKep9E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Test for sentence_to_indices\n",
        "print(\"Text: \" + text_array[0])\n",
        "print(\"Text: \" + text_array[1])\n",
        "test = sentences_to_indices(text_array[:2],word_to_index,10)\n",
        "print(test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UkKL46oOep9P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Test for rectifying the error on one word at 1800 index in embeddings file. The error was rectified in creating embedding layer.\n",
        "# u = index_to_word[1800]\n",
        "# lol = word_to_vec_map[u]\n",
        "# print(np.shape(lol))\n",
        "# lol = np.zeros((100,))\n",
        "# lol[:len(lol)] = word_to_vec_map[u]\n",
        "# print(np.shape(lol))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EzyICeUyep9W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def pretrained_embedding_layer(word_to_vec_map, word_to_index):\n",
        "    \"\"\"\n",
        "    Creates a Keras Embedding() layer and loads in pre-trained GloVe 100-dimensional vectors.\n",
        "    \n",
        "    Arguments:\n",
        "    word_to_vec_map -- dictionary mapping words to their GloVe vector representation.\n",
        "    word_to_index -- dictionary mapping from words to their indices in the vocabulary.\n",
        "\n",
        "    Returns:\n",
        "    embedding_layer -- pretrained layer Keras instance\n",
        "    \"\"\"\n",
        "    \n",
        "    vocab_len = len(word_to_index) + 1                  # adding 1 to fit Keras embedding (requirement)\n",
        "    emb_dim = word_to_vec_map[\"cucumber\"].shape[0]      # define dimensionality of GloVe word vectors\n",
        "    \n",
        "    # Initialize the embedding matrix as a numpy array of zeros of shape\n",
        "    emb_matrix = np.zeros((vocab_len, emb_dim))\n",
        "    \n",
        "    # Set each row \"index\" of the embedding matrix to be the word vector representation of the \"index\"th word of the vocabulary\n",
        "    for word, index in word_to_index.items():\n",
        "        enc = word_to_vec_map[word]\n",
        "        # Handling error in glove \n",
        "        if not (np.shape(enc)[0] == emb_dim):\n",
        "            temp = np.zeros((emb_dim,))\n",
        "            temp[:len(enc)] = word_to_vec_map[word]\n",
        "            enc = temp\n",
        "        # Handling of error complete\n",
        "        emb_matrix[index] = enc\n",
        "\n",
        "    # Define Keras embedding layer\n",
        "    embedding_layer = Embedding(vocab_len, emb_dim, mask_zero = True, trainable = False)\n",
        "\n",
        "    # Build the embedding layer.\n",
        "    embedding_layer.build((None,))\n",
        "    \n",
        "    # Set the weights of the embedding layer to the embedding matrix.\n",
        "    embedding_layer.set_weights([emb_matrix])\n",
        "    \n",
        "    return embedding_layer"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C7oJqF9rep9c",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "embedding_layer = pretrained_embedding_layer(word_to_vec_map, word_to_index)\n",
        "print(\"weights[0][1][3] =\", embedding_layer.get_weights()[0][1][3])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DFWeWm_mrYOl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def softmax(x, axis=1):\n",
        "    ndim = K.ndim(x)\n",
        "    if ndim == 2:\n",
        "        return K.softmax(x)\n",
        "    elif ndim > 2:\n",
        "        e = K.exp(x - K.max(x, axis=axis, keepdims=True))\n",
        "        s = K.sum(e, axis=axis, keepdims=True)\n",
        "        return e / s\n",
        "    else:\n",
        "        raise ValueError('Cannot apply softmax to a tensor that is 1D')\n",
        "        \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EGMUAgryqTSX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Global initialisation for Attention layer. The weights need to be same for all iterations.\n",
        "repeator = RepeatVector(70)\n",
        "concatenator = Concatenate(axis=-1)\n",
        "densor1 = Dense(10, activation = \"tanh\")\n",
        "densor2 = Dense(1, activation = \"relu\")\n",
        "activator = Activation(softmax, name='attention_weights') \n",
        "dotor = Dot(axes = 1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NX8XI4OpoI5J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def one_step_attention(a, s_prev):\n",
        "    \"\"\"\n",
        "    Performs one step of attention: Outputs a context vector computed as a dot product of the attention weights\n",
        "    \"alphas\" and the hidden states \"a\" of the Bi-LSTM.\n",
        "    \"\"\"\n",
        "    \n",
        "    s_prev = repeator(s_prev)\n",
        "    concat = concatenator([a, s_prev])\n",
        "    e = densor1(concat)\n",
        "    energies = densor2(e)\n",
        "    alphas = activator(energies)\n",
        "    context = dotor([alphas,a])\n",
        "    \n",
        "    return context"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZpjvSwaCKDYi",
        "colab_type": "text"
      },
      "source": [
        "<h2> Motivation Classifier</h2>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jvd8bL_hep9i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def Motivation_classifier(input_shape,enc_shape, word_to_vec_map, word_to_index):\n",
        "    \"\"\"\n",
        "    Function creating the model's graph.\n",
        "    \n",
        "    Arguments:\n",
        "    input_shape -- shape of the input\n",
        "    enc_shape -- Size of image features\n",
        "    word_to_vec_map -- dictionary mapping every word in a vocabulary into its 100-dimensional vector representation\n",
        "    word_to_index -- dictionary mapping from words to their indices in the vocabulary\n",
        "\n",
        "    Returns:\n",
        "    model -- a model instance in Keras\n",
        "    \"\"\"\n",
        "    \n",
        "\n",
        "    # Define sentence_indices as the input of the graph.\n",
        "    sentence_indices = Input(shape = input_shape, dtype='int32')\n",
        "    \n",
        "    # Embedding layer pretrained with GloVe Vectors\n",
        "    embedding_layer = pretrained_embedding_layer(word_to_vec_map, word_to_index)\n",
        "    \n",
        "    # Propagating sentence_indices through embedding layer\n",
        "    embeddings = embedding_layer(sentence_indices)\n",
        "    \n",
        "    # Define image_encodings as Input for model\n",
        "    image_enc = Input(shape = enc_shape, dtype='float32')\n",
        "    \n",
        "    # Dense Layers for reducing dimension\n",
        "    s = Dense(200,activation='relu')(image_enc)\n",
        "    temp = s\n",
        "\n",
        "    #Initializing the Initial states of LSTM with images \n",
        "    encoder_states = [s, s]\n",
        "    \n",
        "    # Propagate the embeddings through an LSTM layer with 200-dimensional hidden state\n",
        "    embeddings = Dropout(0.2)(embeddings)\n",
        "    lstm_one = Bidirectional(LSTM(200, return_sequences=True))\n",
        "    \n",
        "    X = lstm_one(embeddings)\n",
        "    X = Dropout(0.4)(X)\n",
        "\n",
        "    ############__Attention__###############\n",
        "    context_conv = []\n",
        "\n",
        "    post_activation_LSTM_cell = LSTM(200, return_state = True, dropout=0.2, recurrent_dropout=0.2)\n",
        "\n",
        "    for t in range(input_shape[0]):\n",
        "    \n",
        "        context = one_step_attention(X, temp)\n",
        "    \n",
        "        s, _, c = post_activation_LSTM_cell(context, initial_state=[s, s])\n",
        "\n",
        "        context_conv.append((s))\n",
        "    ############__Attention__###############\n",
        "\n",
        "    conv = keras.layers.Concatenate()(context_conv)\n",
        "    conv = keras.layers.Reshape((input_shape[0],200))(conv)\n",
        "\n",
        "    X = keras.layers.GRU(64, return_sequences= False, dropout=0.2, recurrent_dropout=0.2)(conv)\n",
        "\n",
        "    X = BatchNormalization(momentum=0.99, epsilon=0.001, center=True, scale=True)(X)\n",
        "    \n",
        "    # Add dropout with a probability of 0.1\n",
        "    X = Dropout(0.1)(X)\n",
        "        \n",
        "    X = Dense(1,activation='elu',kernel_regularizer=keras.regularizers.l2(0.001))(X)\n",
        "\n",
        "    X = Activation('sigmoid')(X)\n",
        "    \n",
        "    # Creating Model instance \n",
        "    model = Model(inputs=[sentence_indices, image_enc], outputs=[X], name=\"motivation_task\")\n",
        "    \n",
        "    \n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EyeaKkPFep9s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_motivate = Motivation_classifier((70,),(2048,), word_to_vec_map, word_to_index)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uD-ntuTBep9w",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_motivate.compile(loss=['binary_crossentropy'], optimizer=\"Adam\", metrics=['acc'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wMYYS92Uep94",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Converting sentences to indices for model input\n",
        "X_train_indices = sentences_to_indices(X, word_to_index, 70)\n",
        "print(np.shape(X_train_indices))\n",
        "X_val_indices = sentences_to_indices(X_val, word_to_index, 70)\n",
        "print(np.shape(X_val_indices))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q0iERWNdep98",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Callbacks for saving weights to pick one with best accuracy\n",
        "from keras.callbacks import ModelCheckpoint\n",
        "checkpoint = ModelCheckpoint(filepath='/content/memotion_motivation.{epoch:03d}.h5',\n",
        "                                 monitor='val_acc',\n",
        "                                 verbose=1,\n",
        "                                 save_best_only=True,\n",
        "                                 save_weights_only=True,\n",
        "                                 period=10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_VoQ43NQmh7_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Finding count of each class\n",
        "count_motivate = Counter(Y_motivate)\n",
        "print(count_motivate)\n",
        "print(count_motivate.keys())\n",
        "motivational = count_motivate[1]\n",
        "not_motivational = count_motivate[0]\n",
        "total = motivational + not_motivational\n",
        "print(\"Motivational: \",motivational)\n",
        "print(\"Not Motivational: \",not_motivational)\n",
        "print(\"Total: \",total)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1YeBLlnomkLE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# class weights calculation\n",
        "class_weight_motivation = {}\n",
        "maxi = max(not_motivational, motivational)\n",
        "weight_for_0 = (maxi / (maxi + not_motivational))\n",
        "weight_for_1 = (maxi / (maxi + motivational))\n",
        "\n",
        "class_weight_motivation = {0: weight_for_0, 1: weight_for_1}\n",
        "\n",
        "print('Weight for class 0: {:.2f}'.format(weight_for_0))\n",
        "print('Weight for class 1: {:.2f}'.format(weight_for_1))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rv3IwQlsep-B",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "history_motivate = model_motivate.fit(\n",
        "    [X_train_indices,image_encodings],\n",
        "    [Y_motivate],\n",
        "    batch_size = 200,\n",
        "    epochs = 120,\n",
        "    validation_data = ([X_val_indices,image_encodings_val],[Y_motivate_val]),\n",
        "    callbacks = [checkpoint],\n",
        "    class_weight = [class_weight_motivation]\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cghqPxGpzvsR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "history = history_motivate\n",
        "import matplotlib.pyplot as plt\n",
        "plt.plot(history.history['acc'])\n",
        "plt.plot(history.history['val_acc'])\n",
        "plt.title('Model accuracy')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train', 'Test'], loc='lower right')\n",
        "plt.show()\n",
        "\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.title('Model loss')\n",
        "plt.ylabel('Loss')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train', 'Test'], loc='lower right')\n",
        "plt.show()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RALAFjAtFepu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Loading saved weights\n",
        "# model_motivate.load_weights('/content/drive/My Drive/Memotion /memotion validation/memotion_motivate.h5')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gKhsEkyU1iXO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "answer_motivate = model_motivate.predict([X_val_indices,image_encodings_val])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dZC5v4Bt1I8n",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pred_motivate = []\n",
        "\n",
        "for i in range(0,len(X_val)):\n",
        "    num = answer_motivate[i]\n",
        "    if(num>0.5):\n",
        "      pred_motivate.append(1)\n",
        "    else:\n",
        "      pred_motivate.append(0)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MNTDKipBsTT_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "con_mat_motivate = tf.math.confusion_matrix(labels=Y_motivate_val, predictions=pred_motivate)\n",
        "con_mat_motivate = tf.Session().run(con_mat_motivate)\n",
        "print(con_mat_motivate)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4TUVX0_WQn05",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import f1_score\n",
        "f1_score(Y_motivate_val, pred_motivate, average='micro')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e55XGwA3C0-t",
        "colab_type": "text"
      },
      "source": [
        "<h2>End of motivation</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_YlWZ-nZTQ7P",
        "colab_type": "text"
      },
      "source": [
        "<h2>Sarcasm</h2>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HmUSFJHXxGqP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "K.clear_session()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ec4gUbWzQwIu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "repeator = RepeatVector(70)\n",
        "concatenator = Concatenate(axis=-1)\n",
        "densor1 = Dense(10, activation = \"tanh\")\n",
        "densor2 = Dense(1, activation = \"relu\")\n",
        "activator = Activation(softmax, name='attention_weights') \n",
        "dotor = Dot(axes = 1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UBiUrpsYQyg6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def Sarcasm_classifier(input_shape,enc_shape, word_to_vec_map, word_to_index):\n",
        "    \"\"\"\n",
        "    Function creating the model's graph.\n",
        "    \n",
        "    Arguments:\n",
        "    input_shape -- shape of the input\n",
        "    enc_shape -- Size of image features\n",
        "    word_to_vec_map -- dictionary mapping every word in a vocabulary into its 100-dimensional vector representation\n",
        "    word_to_index -- dictionary mapping from words to their indices in the vocabulary\n",
        "\n",
        "    Returns:\n",
        "    model -- a model instance in Keras\n",
        "    \"\"\"\n",
        "    \n",
        "\n",
        "    # Define sentence_indices as the input of the graph.\n",
        "    sentence_indices = Input(shape = input_shape, dtype='int32')\n",
        "    \n",
        "    # Embedding layer pretrained with GloVe Vectors\n",
        "    embedding_layer = pretrained_embedding_layer(word_to_vec_map, word_to_index)\n",
        "    embeddings = embedding_layer(sentence_indices)\n",
        "    \n",
        "    # Define image_encodings as Input for model\n",
        "    image_enc = Input(shape = enc_shape, dtype='float32')\n",
        "    \n",
        "    # Dense Layers for reducing dimension\n",
        "    s = Dense(200,activation='relu')(image_enc)\n",
        "    temp = s\n",
        "\n",
        "    #Initializing the hidden states of LSTM with images \n",
        "    encoder_states = [s, s]\n",
        "    \n",
        "    # Propagate the embeddings through an LSTM layer with 200-dimensional hidden state\n",
        "    embeddings = Dropout(0.2)(embeddings)\n",
        "    lstm_one = Bidirectional(LSTM(200, return_sequences=True))\n",
        "    \n",
        "    X = lstm_one(embeddings)\n",
        "    X = Dropout(0.4)(X)\n",
        "\n",
        "    ############__Attention__###############\n",
        "    context_conv = []\n",
        "\n",
        "    post_activation_LSTM_cell = LSTM(200, return_state = True, dropout=0.2, recurrent_dropout=0.2)\n",
        "\n",
        "    for t in range(input_shape[0]):\n",
        "    \n",
        "        context = one_step_attention(X, temp)\n",
        "            \n",
        "        s, _, c = post_activation_LSTM_cell(context, initial_state=[s, s])\n",
        "        \n",
        "        context_conv.append((s))\n",
        "    ############__Attention__###############\n",
        "\n",
        "    conv = keras.layers.Concatenate()(context_conv)\n",
        "    conv = keras.layers.Reshape((input_shape[0],200))(conv)\n",
        "\n",
        "    X = keras.layers.GRU(64, return_sequences= False, dropout=0.2, recurrent_dropout=0.2)(conv)\n",
        "\n",
        "    X = BatchNormalization(momentum=0.99, epsilon=0.001, center=True, scale=True)(X)\n",
        "    \n",
        "    # Add dropout with a probability of 0.1\n",
        "    X = Dropout(0.1)(X)\n",
        "\n",
        "    X = Dense(3,activation='elu',kernel_regularizer=keras.regularizers.l2(0.001))(X)\n",
        "    Y = Dense(4,activation='elu',kernel_regularizer=keras.regularizers.l2(0.001))(X)\n",
        "    \n",
        "    # Add a softmax activation\n",
        "    X = Activation('softmax')(X)\n",
        "    Y = Activation('softmax')(Y)\n",
        "    \n",
        "    # Model instance\n",
        "    model = Model(inputs=[sentence_indices, image_enc], outputs=[X,Y], name=\"sarcasm_task\")\n",
        "    \n",
        "    \n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u5TXj7ZGQydt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_sarcasm = Sarcasm_classifier((70,),(2048,), word_to_vec_map, word_to_index)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pyRkU42YQybQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_sarcasm.compile(loss=['categorical_crossentropy','categorical_crossentropy'], optimizer=\"Adam\", metrics=['acc'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7_tpY7mNQyYg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Converting sentence to indices\n",
        "X_train_indices = sentences_to_indices(X, word_to_index, 70)\n",
        "print(np.shape(X_train_indices))\n",
        "X_val_indices = sentences_to_indices(X_val, word_to_index, 70)\n",
        "print(np.shape(X_val_indices))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n0wXulOsQyWG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.callbacks import ModelCheckpoint\n",
        "checkpoint = ModelCheckpoint(filepath='/content/memotion_sarcasm.{epoch:03d}.h5',\n",
        "                                 monitor='val_activation_2_acc',\n",
        "                                 verbose=1,\n",
        "                                 save_best_only=True,\n",
        "                                 save_weights_only=True,\n",
        "                                 period=10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WjtrPdmxQyPK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Count for each prediction class \n",
        "count_sarcasm = Counter(Yn_sarcasm)\n",
        "print(count_sarcasm)\n",
        "print(count_sarcasm.keys())\n",
        "not_sarcastic = count_sarcasm[0]\n",
        "general = count_sarcasm[1]\n",
        "twisted_meaning = count_sarcasm[2]\n",
        "very_twisted = count_sarcasm[3]\n",
        "\n",
        "total = not_sarcastic + general + very_twisted + twisted_meaning\n",
        "print(\"Not sarcastic: \",not_sarcastic)\n",
        "print(\"General: \",general)\n",
        "print(\"Twisted meaning: \",twisted_meaning)\n",
        "print(\"Very twisted: \",very_twisted)\n",
        "print(\"Total: \",total)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "icsLYto7QyDr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# class weights for quantification layer\n",
        "class_weight_sarcasm_4 = {}\n",
        "maxi = max(not_sarcastic, general, very_twisted, twisted_meaning)\n",
        "weight_for_0 = (maxi / (maxi + not_sarcastic))\n",
        "weight_for_1 = (maxi / (maxi + general))\n",
        "weight_for_2 = (maxi / (maxi + twisted_meaning))\n",
        "weight_for_3 = (maxi / (maxi + very_twisted))\n",
        "\n",
        "class_weight_sarcasm_4 = {0: weight_for_0, 1: weight_for_1, 2: weight_for_2, 3: weight_for_3}\n",
        "\n",
        "print('Weight for class 0: {:.2f}'.format(weight_for_0))\n",
        "print('Weight for class 1: {:.2f}'.format(weight_for_1))\n",
        "print('Weight for class 2: {:.2f}'.format(weight_for_2))\n",
        "print('Weight for class 3: {:.2f}'.format(weight_for_3))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gSiij3yXQx35",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Class weights for classification layer\n",
        "class_weight_sarcasm_3 = {}\n",
        "maxi = max(not_sarcastic, general, very_twisted + twisted_meaning)\n",
        "weight_for_0 = (maxi / (maxi + not_sarcastic))\n",
        "weight_for_1 = (maxi / (maxi + general))\n",
        "weight_for_2 = (maxi / (maxi + twisted_meaning + very_twisted))\n",
        "\n",
        "class_weight_sarcasm_3 = {0: weight_for_0, 1: weight_for_1, 2: weight_for_2}\n",
        "\n",
        "print('Weight for class 0: {:.2f}'.format(weight_for_0))\n",
        "print('Weight for class 1: {:.2f}'.format(weight_for_1))\n",
        "print('Weight for class 2: {:.2f}'.format(weight_for_2))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9-OlJGE_UDOh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "history_sarcasm = model_sarcasm.fit(\n",
        "    [X_train_indices,image_encodings],\n",
        "    [Y_oh_sarcasm_train,Y_oh_n_sarcasm_train],\n",
        "    batch_size = 200,\n",
        "    epochs = 120,\n",
        "    validation_data = ([X_val_indices,image_encodings_val],[Y_oh_sarcasm_val,Y_oh_n_sarcasm_val]),\n",
        "    callbacks = [checkpoint],\n",
        "    class_weight = [class_weight_sarcasm_3,class_weight_sarcasm_4]\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pK1sLDJ3UDMr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "history = history_sarcasm\n",
        "import matplotlib.pyplot as plt\n",
        "plt.plot(history.history['activation_1_acc'])\n",
        "plt.plot(history.history['val_activation_1_acc'])\n",
        "plt.title('Model accuracy')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train-3', 'Test-3'], loc='lower right')\n",
        "plt.show()\n",
        "\n",
        "plt.plot(history.history['activation_1_loss'])\n",
        "plt.plot(history.history['val_activation_1_loss'])\n",
        "plt.title('Model loss')\n",
        "plt.ylabel('Loss')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train-3', 'Test-3'], loc='lower right')\n",
        "plt.show()\n",
        "\n",
        "# ----------------------------------------------------------------\n",
        "plt.plot(history.history['activation_2_acc'])\n",
        "plt.plot(history.history['val_activation_2_acc'])\n",
        "plt.title('Model accuracy')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train-4', 'Test-4'], loc='lower right')\n",
        "plt.show()\n",
        "\n",
        "plt.plot(history.history['activation_2_loss'])\n",
        "plt.plot(history.history['val_activation_2_loss'])\n",
        "plt.title('Model loss')\n",
        "plt.ylabel('Loss')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train-4', 'Test-4'], loc='lower right')\n",
        "plt.show()\n",
        "\n",
        "#----------------------------------------------------------------\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.title('Model loss')\n",
        "plt.ylabel('Loss')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train', 'Test'], loc='lower right')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5FiDqJM_oBcr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Loading saved weights\n",
        "# model_sarcasm.load_weights(\"/content/drive/My Drive/Memotion /memotion validation/memotion_sarcasm.h5\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "URBYzM03UDHm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# answer_sarcasm[0][i] contains results of classification layer\n",
        "# answer_sarcasm[1][i] contains results of quantification layer\n",
        "answer_sarcasm = model_sarcasm.predict([X_val_indices,image_encodings_val])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MapLbY20UC4R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pred_sarcasm = []\n",
        "predn_sarcasm = [] # Quantification result\n",
        "pred2_sarcasm = [] # Classification result\n",
        "\n",
        "for i in range(0,len(X_val)):\n",
        "    pred_sarcasm.append(np.argmax(answer_sarcasm[0][i])) # 3 NEURON OUTPUT\n",
        "    predn_sarcasm.append(np.argmax(answer_sarcasm[1][i])) # 4 NEURON OUTPUT\n",
        "    \n",
        "    # Classification Output using 3 neuron output\n",
        "    num = np.argmax(answer_sarcasm[0][i])\n",
        "    if(num ==0 ):\n",
        "      pred2_sarcasm.append(0)\n",
        "    else:\n",
        "      pred2_sarcasm.append(1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2wM9SwzcmBRl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "con_n_mat_sarcasm = tf.math.confusion_matrix(labels=Yn_sarcasm_val, predictions=predn_sarcasm)\n",
        "con_n_mat_sarcasm = tf.Session().run(con_n_mat_sarcasm)\n",
        "print(con_n_mat_sarcasm)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aYq2QB7LnLZy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import f1_score\n",
        "f1_score(Yn_sarcasm_val, predn_sarcasm, average='micro')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4DPEDeTyyr-n",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "con_nf_mat_sarcasm = tf.math.confusion_matrix(labels=Ynf_sarcasm_val, predictions=pred2_sarcasm)\n",
        "con_nf_mat_sarcasm = tf.Session().run(con_nf_mat_sarcasm)\n",
        "print(con_nf_mat_sarcasm)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0fYsQVSxzFOY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import f1_score\n",
        "f1_score(Ynf_sarcasm_val, pred2_sarcasm, average='micro')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zkWTHjfwnzx5",
        "colab_type": "text"
      },
      "source": [
        "<h2> End of Sarcasm </h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vd34hMacpBkl",
        "colab_type": "text"
      },
      "source": [
        "<h2>Humour</h2>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "10iGEYx5Br9R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "K.clear_session()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U-k5P4aPnLXL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "repeator = RepeatVector(70)\n",
        "concatenator = Concatenate(axis=-1)\n",
        "densor1 = Dense(10, activation = \"tanh\")\n",
        "densor2 = Dense(1, activation = \"relu\")\n",
        "activator = Activation(softmax, name='attention_weights') \n",
        "dotor = Dot(axes = 1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lYMbVwUJnLU1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def Humour_classifier(input_shape,enc_shape, word_to_vec_map, word_to_index):\n",
        "    \"\"\"\n",
        "    Function creating the model's graph.\n",
        "    \n",
        "    Arguments:\n",
        "    input_shape -- shape of the input\n",
        "    enc_shape -- Size of image features\n",
        "    word_to_vec_map -- dictionary mapping every word in a vocabulary into its 100-dimensional vector representation\n",
        "    word_to_index -- dictionary mapping from words to their indices in the vocabulary\n",
        "\n",
        "    Returns:\n",
        "    model -- a model instance in Keras\n",
        "    \"\"\"\n",
        "    \n",
        "\n",
        "    # Define sentence_indices as the input of the graph.\n",
        "    sentence_indices = Input(shape = input_shape, dtype='int32')\n",
        "    \n",
        "    # Embedding layer pretrained with GloVe Vectors\n",
        "    embedding_layer = pretrained_embedding_layer(word_to_vec_map, word_to_index)\n",
        "    embeddings = embedding_layer(sentence_indices)\n",
        "    \n",
        "    # Define image_encodings as Input\n",
        "    image_enc = Input(shape = enc_shape, dtype='float32')\n",
        "    \n",
        "    # Dense Layers for reducing dimension\n",
        "    s = Dense(200,activation='relu')(image_enc)\n",
        "    temp = s\n",
        "\n",
        "    #Initializing the activations of LSTM with images \n",
        "    encoder_states = [s, s]\n",
        "    \n",
        "    # Propagate the embeddings through an LSTM layer with 200-dimensional hidden state.\n",
        "    embeddings = Dropout(0.2)(embeddings)\n",
        "    lstm_one = Bidirectional(LSTM(200, return_sequences=True))\n",
        "    \n",
        "    X = lstm_one(embeddings)\n",
        "    X = Dropout(0.4)(X)\n",
        "\n",
        "    ############__Attention__###############\n",
        "    context_conv = []\n",
        "\n",
        "    post_activation_LSTM_cell = LSTM(200, return_state = True, dropout=0.2, recurrent_dropout=0.2)\n",
        "\n",
        "    for t in range(input_shape[0]):\n",
        "    \n",
        "        context = one_step_attention(X, temp)\n",
        "            \n",
        "        s, _, c = post_activation_LSTM_cell(context, initial_state=[s, s])\n",
        "\n",
        "        context_conv.append((s))\n",
        "    ############__Attention__###############\n",
        "\n",
        "    conv = keras.layers.Concatenate()(context_conv)\n",
        "    conv = keras.layers.Reshape((input_shape[0],200))(conv)\n",
        "\n",
        "    X = keras.layers.GRU(64, return_sequences= False, dropout=0.2, recurrent_dropout=0.2)(conv)\n",
        "\n",
        "    X = BatchNormalization(momentum=0.99, epsilon=0.001, center=True, scale=True)(X)\n",
        "    \n",
        "    # Add dropout with a probability of 0.1\n",
        "    X = Dropout(0.1)(X)\n",
        "\n",
        "    X = Dense(3,activation='elu',kernel_regularizer=keras.regularizers.l2(0.001))(X)\n",
        "    Y = Dense(4,activation='elu',kernel_regularizer=keras.regularizers.l2(0.001))(X)\n",
        "    # Add a softmax activation\n",
        "    X = Activation('softmax')(X)\n",
        "    Y = Activation('softmax')(Y)\n",
        "    \n",
        "    # Create Model instance.\n",
        "    model = Model(inputs=[sentence_indices, image_enc], outputs=[X,Y], name=\"humour_task\")\n",
        "    \n",
        "    \n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y2KWdRQPnLQc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_humour = Humour_classifier((70,),(2048,), word_to_vec_map, word_to_index)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pUyLDfQvnLEe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_humour.compile(loss=['categorical_crossentropy','categorical_crossentropy'], optimizer=\"Adam\", metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W3-zIzNep_8t",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# COnverting sentences to indices\n",
        "X_train_indices = sentences_to_indices(X, word_to_index, 70)\n",
        "print(np.shape(X_train_indices))\n",
        "X_val_indices = sentences_to_indices(X_val, word_to_index, 70)\n",
        "print(np.shape(X_val_indices))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9hribpKfqA3J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Checkpoint callback\n",
        "checkpoint = ModelCheckpoint(filepath='/content/memotion_humour.{epoch:03d}.h5',\n",
        "                                 monitor='val_activation_2_acc',\n",
        "                                 verbose=1,\n",
        "                                 save_best_only=True,\n",
        "                                 save_weights_only=True,\n",
        "                                 period=10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mqNY7SOpqAvG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Count for each class\n",
        "count_humour = Counter(Yn_humour)\n",
        "print(count_humour)\n",
        "print(count_humour.keys())\n",
        "not_funny = count_humour[0]\n",
        "funny = count_humour[1]\n",
        "very_funny = count_humour[2]\n",
        "hilarious = count_humour[3]\n",
        "\n",
        "total = not_funny + funny + very_funny + hilarious\n",
        "print(\"Not funny: \",not_funny)\n",
        "print(\"funny: \",funny)\n",
        "print(\"Very funny: \",very_funny)\n",
        "print(\"Hilarious: \",hilarious)\n",
        "print(\"Total: \",total)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bcZBxL-3qAsE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Class weights for quantification\n",
        "class_weight_humour_4 = {}\n",
        "maxi = max(not_funny, funny, very_funny, hilarious)\n",
        "weight_for_0 = (maxi / (maxi + not_funny))\n",
        "weight_for_1 = (maxi / (maxi + funny))\n",
        "weight_for_2 = (maxi / (maxi + very_funny))\n",
        "weight_for_3 = (maxi / (maxi + hilarious))\n",
        "\n",
        "class_weight_humour_4 = {0: weight_for_0, 1: weight_for_1, 2: weight_for_2, 3: weight_for_3}\n",
        "\n",
        "print('Weight for class 0: {:.2f}'.format(weight_for_0))\n",
        "print('Weight for class 1: {:.2f}'.format(weight_for_1))\n",
        "print('Weight for class 2: {:.2f}'.format(weight_for_2))\n",
        "print('Weight for class 3: {:.2f}'.format(weight_for_3))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "07RuzH9NqlZ-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Class weights for classification\n",
        "class_weight_humour_3 = {}\n",
        "maxi = max(not_funny, funny, very_funny + hilarious)\n",
        "weight_for_0 = (maxi / (maxi + not_funny))\n",
        "weight_for_1 = (maxi / (maxi + funny))\n",
        "weight_for_2 = (maxi / (maxi + very_funny + hilarious))\n",
        "\n",
        "class_weight_humour_3 = {0: weight_for_0, 1: weight_for_1, 2: weight_for_2}\n",
        "\n",
        "print('Weight for class 0: {:.2f}'.format(weight_for_0))\n",
        "print('Weight for class 1: {:.2f}'.format(weight_for_1))\n",
        "print('Weight for class 2: {:.2f}'.format(weight_for_2))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w5--lfsOqlTG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "history_humour = model_humour.fit(\n",
        "    [X_train_indices,image_encodings],\n",
        "    [Y_oh_humour_train,Y_oh_n_humour_train],\n",
        "    batch_size = 200,\n",
        "    epochs = 120,\n",
        "    validation_data = ([X_val_indices,image_encodings_val],[Y_oh_humour_val,Y_oh_n_humour_val]),\n",
        "    callbacks = [checkpoint],\n",
        "    class_weight = [class_weight_humour_3,class_weight_humour_4]\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jaEMlDZ8qlLy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "history = history_humour\n",
        "import matplotlib.pyplot as plt\n",
        "plt.plot(history.history['activation_1_acc'])\n",
        "plt.plot(history.history['val_activation_1_acc'])\n",
        "plt.title('Model accuracy')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train-3', 'Test-3'], loc='lower right')\n",
        "plt.show()\n",
        "\n",
        "plt.plot(history.history['activation_1_loss'])\n",
        "plt.plot(history.history['val_activation_1_loss'])\n",
        "plt.title('Model loss')\n",
        "plt.ylabel('Loss')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train-3', 'Test-3'], loc='lower right')\n",
        "plt.show()\n",
        "\n",
        "# ----------------------------------------------------------------\n",
        "plt.plot(history.history['activation_2_acc'])\n",
        "plt.plot(history.history['val_activation_2_acc'])\n",
        "plt.title('Model accuracy')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train-4', 'Test-4'], loc='lower right')\n",
        "plt.show()\n",
        "\n",
        "plt.plot(history.history['activation_2_loss'])\n",
        "plt.plot(history.history['val_activation_2_loss'])\n",
        "plt.title('Model loss')\n",
        "plt.ylabel('Loss')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train-4', 'Test-4'], loc='lower right')\n",
        "plt.show()\n",
        "\n",
        "#----------------------------------------------------------------\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.title('Model loss')\n",
        "plt.ylabel('Loss')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train', 'Test'], loc='lower right')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3G1ZLJwl57OU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Loading saved weights\n",
        "# model_humour.load_weights(\"/content/drive/My Drive/Memotion /memotion validation/memotion_humour.h5\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sLQlIPGxqlDK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "answer_humour = model_humour.predict([X_val_indices,image_encodings_val])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z-jRZGXq3jZe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pred_humour = []\n",
        "predn_humour = [] # Quantification Result\n",
        "pred2_humour = [] # Classification result\n",
        "\n",
        "for i in range(0,len(X_val)):\n",
        "    pred_humour.append(np.argmax(answer_humour[0][i])) # 3 NEURON OUTPUT\n",
        "    predn_humour.append(np.argmax(answer_humour[1][i])) # 4 NEURON OUTPUT\n",
        "    \n",
        "    # Classification Output using 3 neuron output\n",
        "    num = np.argmax(answer_humour[0][i])\n",
        "    if(num == 0):\n",
        "      pred2_humour.append(0)\n",
        "    else:\n",
        "      pred2_humour.append(1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v_Lce3ep47hR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Quantification\n",
        "con_mat_n_humour = tf.math.confusion_matrix(labels=Yn_humour_val, predictions=predn_humour)\n",
        "con_mat_n_humour = tf.Session().run(con_mat_n_humour)\n",
        "print(con_mat_n_humour)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aOBrInqu5roE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import f1_score\n",
        "f1_score(Yn_humour_val, predn_humour, average='micro')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lGn8k7ty0Ouy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Classification\n",
        "con_mat_nf_humour = tf.math.confusion_matrix(labels=Ynf_humour_val, predictions=pred2_humour)\n",
        "con_mat_nf_humour = tf.Session().run(con_mat_nf_humour)\n",
        "print(con_mat_nf_humour)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r6VbaH400kSo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import f1_score\n",
        "f1_score(Ynf_humour_val, pred2_humour, average='micro')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q8qnM8Xj7qKT",
        "colab_type": "text"
      },
      "source": [
        "<h2> End of humour </h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0zE0Ndb47vxT",
        "colab_type": "text"
      },
      "source": [
        "<h2> Offense </h2>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3wn8073-Tju-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "K.clear_session()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j9qUWNm-5xKx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "repeator = RepeatVector(70)\n",
        "concatenator = Concatenate(axis=-1)\n",
        "densor1 = Dense(10, activation = \"tanh\")\n",
        "densor2 = Dense(1, activation = \"relu\")\n",
        "activator = Activation(softmax, name='attention_weights') \n",
        "dotor = Dot(axes = 1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u4O8qczn8NLH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def Offense_classifier(input_shape,enc_shape, word_to_vec_map, word_to_index):\n",
        "    \"\"\"\n",
        "    Function creating the model's graph.\n",
        "    \n",
        "    Arguments:\n",
        "    input_shape -- shape of the input\n",
        "    enc_shape -- Size of image features\n",
        "    word_to_vec_map -- dictionary mapping every word in a vocabulary into its 100-dimensional vector representation\n",
        "    word_to_index -- dictionary mapping from words to their indices in the vocabulary\n",
        "\n",
        "    Returns:\n",
        "    model -- a model instance in Keras\n",
        "    \"\"\"\n",
        "    \n",
        "    # Define sentence_indices as the input of the graph.\n",
        "    sentence_indices = Input(shape = input_shape, dtype='int32')\n",
        "    \n",
        "    # Embedding layer pretrained with GloVe Vectors\n",
        "    embedding_layer = pretrained_embedding_layer(word_to_vec_map, word_to_index)\n",
        "    embeddings = embedding_layer(sentence_indices)\n",
        "    \n",
        "    # Define image_encodings as Input for the model\n",
        "    image_enc = Input(shape = enc_shape, dtype='float32')\n",
        "    \n",
        "    # Dense Layers for reducing dimension\n",
        "    s = Dense(200,activation='relu')(image_enc)\n",
        "    temp = s\n",
        "\n",
        "    #Initializing the hidden states of LSTM with images \n",
        "    encoder_states = [s, s]\n",
        "    \n",
        "    # Propagate the embeddings through an LSTM layer with 200-dimensional hidden state\n",
        "    embeddings = Dropout(0.2)(embeddings)\n",
        "    lstm_one = Bidirectional(LSTM(200, return_sequences=True))\n",
        "    \n",
        "    X = lstm_one(embeddings)\n",
        "    X = Dropout(0.4)(X)\n",
        "\n",
        "    ############__Attention__###############\n",
        "    context_conv = []\n",
        "\n",
        "    post_activation_LSTM_cell = LSTM(200, return_state = True, dropout=0.3, recurrent_dropout=0.2)\n",
        "\n",
        "    for t in range(input_shape[0]):\n",
        "    \n",
        "        context = one_step_attention(X, temp)\n",
        "            \n",
        "        s, _, c = post_activation_LSTM_cell(context, initial_state=[s, s])\n",
        "\n",
        "        context_conv.append((s))\n",
        "    ############__Attention__###############\n",
        "\n",
        "    conv = keras.layers.Concatenate()(context_conv)\n",
        "    conv = keras.layers.Reshape((input_shape[0],200))(conv)\n",
        "\n",
        "    X = keras.layers.GRU(64, return_sequences= False, dropout=0.2, recurrent_dropout=0.2)(conv)\n",
        "\n",
        "    X = BatchNormalization(momentum=0.99, epsilon=0.001, center=True, scale=True)(X)\n",
        "    \n",
        "    # Add dropout with a probability of 0.1\n",
        "    X = Dropout(0.1)(X)\n",
        "    \n",
        "    X = Dense(3,activation='elu',kernel_regularizer=keras.regularizers.l2(0.001))(X)\n",
        "    Y = Dense(4,activation='elu',kernel_regularizer=keras.regularizers.l2(0.001))(X)\n",
        "    # Add a softmax activation\n",
        "    X = Activation('softmax')(X)\n",
        "    Y = Activation('softmax')(Y)\n",
        "    \n",
        "    # Model instance \n",
        "    model = Model(inputs=[sentence_indices, image_enc], outputs=[X,Y], name=\"offense_task\")\n",
        "    \n",
        "    \n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DIMcbP2X8bMN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_offense = Offense_classifier((70,),(2048,), word_to_vec_map, word_to_index)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iiOvtCoR8onR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_offense.compile(loss=['categorical_crossentropy','categorical_crossentropy'], optimizer=\"Adam\", metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_5_UYCiv8yNx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Convert sentences to indices\n",
        "X_train_indices = sentences_to_indices(X, word_to_index, 70)\n",
        "print(np.shape(X_train_indices))\n",
        "X_val_indices = sentences_to_indices(X_val, word_to_index, 70)\n",
        "print(np.shape(X_val_indices))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "689f8HSf88ah",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Checkpointing weights\n",
        "from keras.callbacks import ModelCheckpoint\n",
        "checkpoint = ModelCheckpoint(filepath='/content/memotion_offense.{epoch:03d}.h5',\n",
        "                                 monitor='val_activation_2_acc',\n",
        "                                 verbose=1,\n",
        "                                 save_best_only=True,\n",
        "                                 save_weights_only=True,\n",
        "                                 period=10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6RsrrMAe9FAF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# COunt for each class\n",
        "count_offense = Counter(Yn_offense)\n",
        "print(count_offense)\n",
        "print(count_offense.keys())\n",
        "not_offensive = count_offense[0]\n",
        "slight = count_offense[1]\n",
        "very_offensive = count_offense[2]\n",
        "hateful_offensive = count_offense[3]\n",
        "\n",
        "total = not_offensive + slight + very_offensive + hateful_offensive\n",
        "print(\"Not offensive: \",not_offensive)\n",
        "print(\"slight: \",slight)\n",
        "print(\"Very offensive: \",very_offensive)\n",
        "print(\"Hateful offensive: \",hateful_offensive)\n",
        "print(\"Total: \",total)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N2LlLsoh920B",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Class weights for quantification\n",
        "class_weight_offense_4 = {}\n",
        "maxi = max(not_offensive, slight, very_offensive, hateful_offensive)\n",
        "weight_for_0 = (maxi / (maxi + not_offensive))\n",
        "weight_for_1 = (maxi / (maxi + slight))\n",
        "weight_for_2 = (maxi / (maxi + very_offensive))\n",
        "weight_for_3 = (maxi / (maxi + hateful_offensive))\n",
        "\n",
        "class_weight_offense_4 = {0: weight_for_0, 1: weight_for_1, 2: weight_for_2, 3: weight_for_3}\n",
        "\n",
        "print('Weight for class 0: {:.2f}'.format(weight_for_0))\n",
        "print('Weight for class 1: {:.2f}'.format(weight_for_1))\n",
        "print('Weight for class 2: {:.2f}'.format(weight_for_2))\n",
        "print('Weight for class 3: {:.2f}'.format(weight_for_3))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fJ9RbtG3-XQf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Class weights for classification\n",
        "class_weight_offense_3 = {}\n",
        "maxi = max(not_offensive, slight, very_offensive + hateful_offensive)\n",
        "weight_for_0 = (maxi / (maxi + not_offensive))\n",
        "weight_for_1 = (maxi / (maxi + slight))\n",
        "weight_for_2 = (maxi / (maxi + very_offensive + hateful_offensive))\n",
        "\n",
        "class_weight_offense_3 = {0: weight_for_0, 1: weight_for_1, 2: weight_for_2}\n",
        "\n",
        "print('Weight for class 0: {:.2f}'.format(weight_for_0))\n",
        "print('Weight for class 1: {:.2f}'.format(weight_for_1))\n",
        "print('Weight for class 2: {:.2f}'.format(weight_for_2))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bd3fQcNt-0FV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "history_offense = model_offense.fit(\n",
        "    [X_train_indices,image_encodings],\n",
        "    [Y_oh_offense_train,Y_oh_n_offense_train],\n",
        "    batch_size = 200,\n",
        "    epochs = 120,\n",
        "    validation_data = ([X_val_indices,image_encodings_val],[Y_oh_offense_val,Y_oh_n_offense_val]),\n",
        "    callbacks = [checkpoint],\n",
        "    class_weight = [class_weight_offense_3,class_weight_offense_4]\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kgSK9myo_pb9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "history = history_offense\n",
        "import matplotlib.pyplot as plt\n",
        "plt.plot(history.history['activation_1_acc'])\n",
        "plt.plot(history.history['val_activation_1_acc'])\n",
        "plt.title('Model accuracy')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train-3', 'Test-3'], loc='lower right')\n",
        "plt.show()\n",
        "\n",
        "plt.plot(history.history['activation_1_loss'])\n",
        "plt.plot(history.history['val_activation_1_loss'])\n",
        "plt.title('Model loss')\n",
        "plt.ylabel('Loss')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train-3', 'Test-3'], loc='lower right')\n",
        "plt.show()\n",
        "\n",
        "# ----------------------------------------------------------------\n",
        "plt.plot(history.history['activation_2_acc'])\n",
        "plt.plot(history.history['val_activation_2_acc'])\n",
        "plt.title('Model accuracy')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train-4', 'Test-4'], loc='lower right')\n",
        "plt.show()\n",
        "\n",
        "plt.plot(history.history['activation_2_loss'])\n",
        "plt.plot(history.history['val_activation_2_loss'])\n",
        "plt.title('Model loss')\n",
        "plt.ylabel('Loss')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train-4', 'Test-4'], loc='lower right')\n",
        "plt.show()\n",
        "\n",
        "#----------------------------------------------------------------\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.title('Model loss')\n",
        "plt.ylabel('Loss')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train', 'Test'], loc='lower right')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "onJRy0I7SeAJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Loading saved weights\n",
        "# model_offense.load_weights(\"/content/drive/My Drive/Memotion /memotion validation/memotion_offense.h5\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ytUxR13uLevD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "answer_offense = model_offense.predict([X_val_indices,image_encodings_val])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4HvydDp5ME-n",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pred_offense = []\n",
        "predn_offense = [] # Quantification results\n",
        "pred2_offense = [] # Classification results\n",
        "\n",
        "for i in range(0,len(X_val)):\n",
        "    pred_offense.append(np.argmax(answer_offense[0][i]))\n",
        "    predn_offense.append(np.argmax(answer_offense[1][i]))\n",
        "    num = np.argmax(answer_offense[0][i])\n",
        "    if(num == 0):\n",
        "      pred2_offense.append(0)\n",
        "    else:\n",
        "      pred2_offense.append(1)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JS83XrwIOiZW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Quantification\n",
        "con_mat_n_offense = tf.math.confusion_matrix(labels=Yn_offense_val, predictions=predn_offense)\n",
        "con_mat_n_offense = tf.Session().run(con_mat_n_offense)\n",
        "print(con_mat_n_offense)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SXaLS4RRPGXu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import f1_score\n",
        "f1_score(Yn_offense_val, predn_offense, average='micro')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d1HjCq_C1oVN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Classification\n",
        "con_mat_nf_offense = tf.math.confusion_matrix(labels=Ynf_offense_val, predictions=pred2_offense)\n",
        "con_mat_nf_offense = tf.Session().run(con_mat_nf_offense)\n",
        "print(con_mat_nf_offense)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FCQ6Wh-S1slp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import f1_score\n",
        "f1_score(Ynf_offense_val, pred2_offense, average='micro')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZdHjyoW9R8a8",
        "colab_type": "text"
      },
      "source": [
        "<h2> End of Offense </h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9zQcfGa3SEfg",
        "colab_type": "text"
      },
      "source": [
        "<h2>Subtask A</h2>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X31helSwUl3Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "K.clear_session()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_hIXQGDgPMA_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "repeator = RepeatVector(70)\n",
        "concatenator = Concatenate(axis=-1)\n",
        "densor1 = Dense(20, activation = \"tanh\")\n",
        "densor2 = Dense(1, activation = \"relu\")\n",
        "activator = Activation(softmax, name='attention_weights') \n",
        "dotor = Dot(axes = 1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "izerSgI0T4Py",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def Mood_classifier(input_shape,enc_shape, word_to_vec_map, word_to_index):\n",
        "    \"\"\"\n",
        "    Function creating the model's graph.\n",
        "    \n",
        "    Arguments:\n",
        "    input_shape -- shape of the input\n",
        "    enc_shape -- Size of image features\n",
        "    word_to_vec_map -- dictionary mapping every word in a vocabulary into its 100-dimensional vector representation\n",
        "    word_to_index -- dictionary mapping from words to their indices in the vocabulary\n",
        "\n",
        "    Returns:\n",
        "    model -- a model instance in Keras\n",
        "    \"\"\"\n",
        "\n",
        "    # Define sentence_indices as the input of the graph.\n",
        "    sentence_indices = Input(shape = input_shape, dtype='int32')\n",
        "    \n",
        "    # Embedding layer pretrained with GloVe Vectors\n",
        "    embedding_layer = pretrained_embedding_layer(word_to_vec_map, word_to_index)\n",
        "    embeddings = embedding_layer(sentence_indices)\n",
        "    \n",
        "    # Define image_encodings as Input for model\n",
        "    image_enc = Input(shape = enc_shape, dtype='float32')\n",
        "    \n",
        "    # Dense Layers for reducing dimension\n",
        "    s = Dense(200,activation='relu')(image_enc)\n",
        "    temp = s\n",
        "\n",
        "    #Initializing the hidden states of LSTM with images \n",
        "    encoder_states = [s, s]\n",
        "    \n",
        "    # Propagate the embeddings through an LSTM layer with 200-dimensional hidden state\n",
        "    embeddings = Dropout(0.2)(embeddings)\n",
        "    lstm_one = Bidirectional(LSTM(200, return_sequences=True))\n",
        "    \n",
        "    X = lstm_one(embeddings)\n",
        "    X = Dropout(0.4)(X)\n",
        "\n",
        "    ############__Attention__###############\n",
        "    context_conv = []\n",
        "\n",
        "    post_activation_LSTM_cell = LSTM(200, return_state = True, dropout=0.2, recurrent_dropout=0.2)\n",
        "\n",
        "    for t in range(input_shape[0]):\n",
        "    \n",
        "        context = one_step_attention(X, temp)\n",
        "            \n",
        "        s, _, c = post_activation_LSTM_cell(context, initial_state=[s, s])\n",
        "\n",
        "        context_conv.append((s))\n",
        "    ############__Attention__###############\n",
        "\n",
        "    conv = keras.layers.Concatenate()(context_conv)\n",
        "    conv = keras.layers.Reshape((input_shape[0],200))(conv)\n",
        "\n",
        "    X = keras.layers.GRU(64, return_sequences= False, dropout=0.2, recurrent_dropout=0.2)(conv)\n",
        "    \n",
        "    X = BatchNormalization(momentum=0.99, epsilon=0.001, center=True, scale=True)(X)\n",
        "  \n",
        "    # Add dropout with a probability of 0.1\n",
        "    X = Dropout(0.1)(X)\n",
        "  \n",
        "    X = Dense(3,activation='elu',kernel_regularizer=keras.regularizers.l2(0.001))(X)\n",
        "    Y = Dense(5,activation='elu',kernel_regularizer=keras.regularizers.l2(0.001))(X)\n",
        "    # Add a softmax activation\n",
        "    X = Activation('softmax')(X)\n",
        "    Y = Activation('softmax')(Y)\n",
        "    \n",
        "    # Model instance\n",
        "    model = Model(inputs=[sentence_indices, image_enc], outputs=[X,Y], name=\"sentiment_task\")\n",
        "    \n",
        "    \n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vvIQzAAFUFNq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_mood = Mood_classifier((70,),(2048,), word_to_vec_map, word_to_index)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xeat5M2GUQOL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_mood.compile(loss=['categorical_crossentropy','categorical_crossentropy'], optimizer=\"Adam\", metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UIry7oZuUZRu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Converting sentence to indices\n",
        "X_train_indices = sentences_to_indices(X, word_to_index, 70)\n",
        "print(np.shape(X_train_indices))\n",
        "X_val_indices = sentences_to_indices(X_val, word_to_index, 70)\n",
        "print(np.shape(X_val_indices))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "br9lM2L7Uhnw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Checkpointing weights\n",
        "from keras.callbacks import ModelCheckpoint\n",
        "checkpoint = ModelCheckpoint(filepath='/content/memotion_sentiment.{epoch:03d}.h5',\n",
        "                                 monitor='val_activation_3_acc',\n",
        "                                 verbose=1,\n",
        "                                 save_best_only=True,\n",
        "                                 save_weights_only=True,\n",
        "                                 period=10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "137jAXXBUugz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Count of each class\n",
        "count_mood = Counter(Yn)\n",
        "print(count_mood)\n",
        "print(count_mood.keys())\n",
        "neutral = count_mood[0]\n",
        "positive = count_mood[1]\n",
        "very_positive = count_mood[2]\n",
        "negative = count_mood[3]\n",
        "very_negative = count_mood[4]\n",
        "\n",
        "total = neutral + positive + very_positive + negative + very_negative\n",
        "print(\"Neutral: \",neutral)\n",
        "print(\"Positive: \",positive)\n",
        "print(\"Very positive: \",very_positive)\n",
        "print(\"Negative: \",negative)\n",
        "print(\"Very negative: \",very_negative)\n",
        "print(\"Total: \",total)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Hs05fYWVtJK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Class weights for quantification\n",
        "class_weight_mood_4 = {}\n",
        "maxi = max(neutral, positive, very_positive, negative, very_negative)\n",
        "weight_for_0 = (maxi / (maxi + neutral))\n",
        "weight_for_1 = (maxi / (maxi + positive))\n",
        "weight_for_2 = (maxi / (maxi + very_positive))\n",
        "weight_for_3 = (maxi / (maxi + negative))\n",
        "weight_for_4 = (maxi / (maxi + very_negative))\n",
        "\n",
        "class_weight_mood_4 = {0: weight_for_0, 1: weight_for_1, 2: weight_for_2, 3: weight_for_3, 4: weight_for_4}\n",
        "\n",
        "print('Weight for class 0: {:.2f}'.format(weight_for_0))\n",
        "print('Weight for class 1: {:.2f}'.format(weight_for_1))\n",
        "print('Weight for class 2: {:.2f}'.format(weight_for_2))\n",
        "print('Weight for class 3: {:.2f}'.format(weight_for_3))\n",
        "print('Weight for class 4: {:.2f}'.format(weight_for_4))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1vNzXeLNWTad",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Class weighhts for classification\n",
        "class_weight_mood_3 = {}\n",
        "maxi = max(neutral, positive, very_positive, negative, very_negative)\n",
        "weight_for_0 = (maxi / (maxi + neutral))\n",
        "weight_for_1 = (maxi / (maxi + positive + very_positive))\n",
        "weight_for_2 = (maxi / (maxi + negative + very_negative))\n",
        "\n",
        "class_weight_mood_3 = {0: weight_for_0, 1: weight_for_1, 2: weight_for_2}\n",
        "\n",
        "print('Weight for class 0: {:.2f}'.format(weight_for_0))\n",
        "print('Weight for class 1: {:.2f}'.format(weight_for_1))\n",
        "print('Weight for class 2: {:.2f}'.format(weight_for_2))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3IY2PyzxWxwa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "history_mood = model_mood.fit(\n",
        "    [X_train_indices,image_encodings],\n",
        "    [Y_oh_train,Y_oh_n_train],\n",
        "    batch_size = 256,\n",
        "    epochs = 150,\n",
        "    validation_data = ([X_val_indices,image_encodings_val],[Y_oh_val,Y_oh_n_val]),\n",
        "    callbacks = [checkpoint],\n",
        "    class_weight = [class_weight_mood_3,class_weight_mood_4]\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FMz1nTilXa0g",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "history = history_mood\n",
        "import matplotlib.pyplot as plt\n",
        "plt.plot(history.history['activation_1_acc'])\n",
        "plt.plot(history.history['val_activation_1_acc'])\n",
        "plt.title('Model accuracy')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train-3', 'Test-3'], loc='lower right')\n",
        "plt.show()\n",
        "\n",
        "plt.plot(history.history['activation_1_loss'])\n",
        "plt.plot(history.history['val_activation_1_loss'])\n",
        "plt.title('Model loss')\n",
        "plt.ylabel('Loss')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train-3', 'Test-3'], loc='lower right')\n",
        "plt.show()\n",
        "\n",
        "# ----------------------------------------------------------------\n",
        "plt.plot(history.history['activation_2_acc'])\n",
        "plt.plot(history.history['val_activation_2_acc'])\n",
        "plt.title('Model accuracy')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train-5', 'Test-5'], loc='lower right')\n",
        "plt.show()\n",
        "\n",
        "plt.plot(history.history['activation_2_loss'])\n",
        "plt.plot(history.history['val_activation_2_loss'])\n",
        "plt.title('Model loss')\n",
        "plt.ylabel('Loss')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train-5', 'Test-5'], loc='lower right')\n",
        "plt.show()\n",
        "\n",
        "#----------------------------------------------------------------\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.title('Model loss')\n",
        "plt.ylabel('Loss')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train', 'Test'], loc='lower right')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "skafF-i3SSLp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Loading saved weights\n",
        "# model_mood.load_weights(\"/content/drive/My Drive/Memotion /memotion validation/memotion_mood.h5\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xg0cleDsFUJg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "answer_mood = model_mood.predict([X_val_indices,image_encodings_val])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8GLkkU2bM56t",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pred_mood = [] # Classification results\n",
        "predn_mood = [] # Quantification results\n",
        "\n",
        "temp = 0\n",
        "for i in range(0,len(X_val)):\n",
        "    pred_mood.append(np.argmax(answer_mood[0][i]))\n",
        "    predn_mood.append(np.argmax(answer_mood[1][i]))\n",
        "    num = np.argmax(answer_mood[0][i])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VrWRVvwTNXTI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Classification results\n",
        "con_mat_mood = tf.math.confusion_matrix(labels=Y_val, predictions=pred_mood)\n",
        "con_mat_mood = tf.Session().run(con_mat_mood)\n",
        "print(con_mat_mood)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o2yZq_5ANz0h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import f1_score\n",
        "f1_score(Y_val, pred_mood, average='micro')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ME1za4B6N9fr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Quantification results\n",
        "con_mat_n_mood = tf.math.confusion_matrix(labels=Yn_val, predictions=predn_mood)\n",
        "con_mat_n_mood = tf.Session().run(con_mat_n_mood)\n",
        "print(con_mat_n_mood)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cZ1-Qs58OYC0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import f1_score\n",
        "f1_score(Yn_val, predn_mood, average='macro')"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}