train_cheat_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "301bbf36",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /Users/teddyoweh/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /Users/teddyoweh/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     /Users/teddyoweh/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /Users/teddyoweh/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    " \n",
    "import re, string \n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import SnowballStemmer\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "nltk.download('punkt')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('stopwords')\n",
    "\n",
    "#for model-building\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import classification_report, f1_score, accuracy_score, confusion_matrix\n",
    "from sklearn.metrics import roc_curve, auc, roc_auc_score\n",
    "\n",
    "# bag of words\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "#for word embedding\n",
    "import gensim\n",
    "from gensim.models import Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "575e5edb",
   "metadata": {},
   "outputs": [],
   "source": [
    "cheat_path='./data/cheat.txt'\n",
    "clean_path='./data/clean.txt'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "84715384",
   "metadata": {},
   "outputs": [],
   "source": [
    "from data import load_data, analyze\n",
    "df_train = load_data(cheat_path,clean_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "d20f87f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Can you give me the answers to this homework a...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I'm struggling with this test, can you help me...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Do you know where I can find the answers to th...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I'm really stuck on this homework, can you jus...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Is there any way you can send me the answers t...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235</th>\n",
       "      <td>Have you started studying for midterms yet?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236</th>\n",
       "      <td>I'm thinking about getting involved in communi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>Have you started thinking about your post-grad...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238</th>\n",
       "      <td>I'm thinking about joining a study abroad prog...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>How do you think about your post-graduation pl...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>240 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  text  target\n",
       "0    Can you give me the answers to this homework a...       1\n",
       "1    I'm struggling with this test, can you help me...       1\n",
       "2    Do you know where I can find the answers to th...       1\n",
       "3    I'm really stuck on this homework, can you jus...       1\n",
       "4    Is there any way you can send me the answers t...       1\n",
       "..                                                 ...     ...\n",
       "235        Have you started studying for midterms yet?       0\n",
       "236  I'm thinking about getting involved in communi...       0\n",
       "237  Have you started thinking about your post-grad...       0\n",
       "238  I'm thinking about joining a study abroad prog...       0\n",
       "239  How do you think about your post-graduation pl...       0\n",
       "\n",
       "[240 rows x 2 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "a5dabd13",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    120\n",
      "0    120\n",
      "Name: target, dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "text      0\n",
       "target    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x=df_train['target'].value_counts()\n",
    "\n",
    "print(x)\n",
    "df_train.isna().sum()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "941a9c7b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word Count\n",
      "Cheat Sentences:  18.6\n",
      "Clean Sentences:  16.966666666666665\n",
      "\n",
      "Character Count\n",
      "Cheat Sentences:  89.44166666666666\n",
      "Clean Sentences:  94.175\n",
      "\n",
      "Unique Word Count\n",
      "Cheat Sentences:  17.291666666666668\n",
      "Clean Sentences:  16.166666666666668\n"
     ]
    }
   ],
   "source": [
    "analyze(df_train) # Mean Data\n",
    "     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "6894cf74",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test sd e a\n"
     ]
    }
   ],
   "source": [
    "text = \"  test sd e%: , ?, ''  a     .  \"\n",
    "\n",
    "def preprocess(text):\n",
    "    text = text.lower() \n",
    "    text=text.strip() \n",
    "    text=re.compile('<.*?>').sub('', text)  \n",
    "    text = re.compile('[%s]' % re.escape(string.punctuation)).sub(' ', text)  \n",
    "    text = re.sub('\\s+', ' ', text)   \n",
    "    text = re.sub(r'\\[[0-9]*\\]',' ',text) \n",
    "    text=re.sub(r'[^\\w\\s]', '', str(text).lower().strip())\n",
    "    text = re.sub(r'\\d',' ',text)  \n",
    "    text = re.sub(r'\\s+',' ',text)  \n",
    "    \n",
    "    return text\n",
    "\n",
    "text=preprocess(text)\n",
    "print(text)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "42dbbac7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test sd e\n",
      "test sd e\n",
      "test sd e\n"
     ]
    }
   ],
   "source": [
    "def stopword(string):\n",
    "    a= [i for i in string.split() if i not in stopwords.words('english')]\n",
    "    return ' '.join(a)\n",
    "\n",
    "text=stopword(text)\n",
    "print(text)\n",
    " \n",
    "    \n",
    "snow = SnowballStemmer('english')\n",
    "def stemming(string):\n",
    "    a=[snow.stem(i) for i in word_tokenize(string) ]\n",
    "    return \" \".join(a)\n",
    "text=stemming(text)\n",
    "print(text)\n",
    "\n",
    " \n",
    "wl = WordNetLemmatizer()\n",
    " \n",
    "def get_wordnet_pos(tag):\n",
    "    if tag.startswith('J'):\n",
    "        return wordnet.ADJ\n",
    "    elif tag.startswith('V'):\n",
    "        return wordnet.VERB\n",
    "    elif tag.startswith('N'):\n",
    "        return wordnet.NOUN\n",
    "    elif tag.startswith('R'):\n",
    "        return wordnet.ADV\n",
    "    else:\n",
    "        return wordnet.NOUN\n",
    "\n",
    "def lemmatizer(string):\n",
    "    word_pos_tags = nltk.pos_tag(word_tokenize(string)) \n",
    "    a=[wl.lemmatize(tag[0], get_wordnet_pos(tag[1])) for idx, tag in enumerate(word_pos_tags)]  \n",
    "    return \" \".join(a)\n",
    "\n",
    "text = lemmatizer(text)\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "807d4d6a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "      <th>clean_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Can you give me the answers to this homework a...</td>\n",
       "      <td>1</td>\n",
       "      <td>give answer homework assignment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I'm struggling with this test, can you help me...</td>\n",
       "      <td>1</td>\n",
       "      <td>struggle test help answer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Do you know where I can find the answers to th...</td>\n",
       "      <td>1</td>\n",
       "      <td>know find answer quiz</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I'm really stuck on this homework, can you jus...</td>\n",
       "      <td>1</td>\n",
       "      <td>really stuck homework give answer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Is there any way you can send me the answers t...</td>\n",
       "      <td>1</td>\n",
       "      <td>way send answer exam</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  target  \\\n",
       "0  Can you give me the answers to this homework a...       1   \n",
       "1  I'm struggling with this test, can you help me...       1   \n",
       "2  Do you know where I can find the answers to th...       1   \n",
       "3  I'm really stuck on this homework, can you jus...       1   \n",
       "4  Is there any way you can send me the answers t...       1   \n",
       "\n",
       "                          clean_text  \n",
       "0    give answer homework assignment  \n",
       "1          struggle test help answer  \n",
       "2              know find answer quiz  \n",
       "3  really stuck homework give answer  \n",
       "4               way send answer exam  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def finalpreprocess(string):\n",
    "    return lemmatizer(stopword(preprocess(string)))\n",
    "\n",
    "df_train['clean_text'] = df_train['text'].apply(lambda x: finalpreprocess(x))\n",
    "df_train=df_train.drop(columns=['word_count','char_count','unique_word_count'])\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "a4a4483e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['clean_text_tok']=[nltk.word_tokenize(i) for i in df_train['clean_text']] \n",
    "model = Word2Vec(df_train['clean_text_tok'],min_count=1) \n",
    " \n",
    "\n",
    "\n",
    "w2v = dict(zip(model.wv.index_to_key, model.wv.vectors))  \n",
    "#fix the syn0 error stuff\n",
    "class MeanEmbeddingVectorizer(object):\n",
    "    def __init__(self, word2vec):\n",
    "        self.word2vec = word2vec\n",
    " \n",
    "        self.dim = len(next(iter(word2vec.values())))\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        return self\n",
    "\n",
    "    def transform(self, X):\n",
    "        return np.array([\n",
    "            np.mean([self.word2vec[w] for w in words if w in self.word2vec]\n",
    "                    or [np.zeros(self.dim)], axis=0)\n",
    "            for words in X\n",
    "        ])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "11bd765e",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_val, y_train, y_val = train_test_split(df_train[\"clean_text\"],\n",
    "                                                  df_train[\"target\"],\n",
    "                                                  test_size=0.2,\n",
    "                                                  shuffle=True)\n",
    "X_train_tok= [nltk.word_tokenize(i) for i in X_train]  #for word2vec\n",
    "X_val_tok= [nltk.word_tokenize(i) for i in X_val]      #for word2vec\n",
    "\n",
    " \n",
    "tfidf_vectorizer = TfidfVectorizer(use_idf=True)\n",
    "X_train_vectors_tfidf = tfidf_vectorizer.fit_transform(X_train)  \n",
    "X_val_vectors_tfidf = tfidf_vectorizer.transform(X_val)  \n",
    " \n",
    "modelw = MeanEmbeddingVectorizer(w2v)\n",
    "X_train_vectors_w2v = modelw.transform(X_train_tok)\n",
    "X_val_vectors_w2v = modelw.transform(X_val_tok)\n",
    "with open('tfidf_vectorizer.pkl', 'wb') as f:\n",
    "    pickle.dump(tfidf_vectorizer, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "bc6b1c1f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.96      0.98        25\n",
      "           1       0.96      1.00      0.98        23\n",
      "\n",
      "    accuracy                           0.98        48\n",
      "   macro avg       0.98      0.98      0.98        48\n",
      "weighted avg       0.98      0.98      0.98        48\n",
      "\n",
      "Confusion Matrix: [[24  1]\n",
      " [ 0 23]]\n",
      "AUC: 1.0\n"
     ]
    }
   ],
   "source": [
    "lr_tfidf=LogisticRegression(solver = 'liblinear', C=10, penalty = 'l2')\n",
    "lr_tfidf.fit(X_train_vectors_tfidf, y_train)  #model\n",
    "\n",
    " \n",
    "y_predict = lr_tfidf.predict(X_val_vectors_tfidf)\n",
    "y_prob = lr_tfidf.predict_proba(X_val_vectors_tfidf)[:,1]\n",
    " \n",
    "\n",
    "print(classification_report(y_val,y_predict))\n",
    "print('Confusion Matrix:',confusion_matrix(y_val, y_predict))\n",
    " \n",
    "fpr, tpr, thresholds = roc_curve(y_val, y_prob)\n",
    "roc_auc = auc(fpr, tpr)\n",
    "print('AUC:', roc_auc)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "abdb7501",
   "metadata": {},
   "outputs": [],
   "source": [
    "### goooodd resultss!!! naive bayes next"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "8e7d9311",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.96      0.98        25\n",
      "           1       0.96      1.00      0.98        23\n",
      "\n",
      "    accuracy                           0.98        48\n",
      "   macro avg       0.98      0.98      0.98        48\n",
      "weighted avg       0.98      0.98      0.98        48\n",
      "\n",
      "Confusion Matrix: [[24  1]\n",
      " [ 0 23]]\n",
      "AUC: 1.0\n"
     ]
    }
   ],
   "source": [
    "nb_tfidf = MultinomialNB()\n",
    "nb_tfidf.fit(X_train_vectors_tfidf, y_train)  #model\n",
    "#cheat_model\n",
    "#Predict y value for test dataset\n",
    "y_predict = nb_tfidf.predict(X_val_vectors_tfidf)\n",
    "y_prob = nb_tfidf.predict_proba(X_val_vectors_tfidf)[:,1]\n",
    " \n",
    "\n",
    "print(classification_report(y_val,y_predict))\n",
    "print('Confusion Matrix:',confusion_matrix(y_val, y_predict))\n",
    " \n",
    "fpr, tpr, thresholds = roc_curve(y_val, y_prob)\n",
    "roc_auc = auc(fpr, tpr)\n",
    "print('AUC:', roc_auc)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "6d335b68",
   "metadata": {},
   "outputs": [],
   "source": [
    "#perfect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "f6acf383",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98        25\n",
      "           1       1.00      0.96      0.98        23\n",
      "\n",
      "    accuracy                           0.98        48\n",
      "   macro avg       0.98      0.98      0.98        48\n",
      "weighted avg       0.98      0.98      0.98        48\n",
      "\n",
      "Confusion Matrix: [[25  0]\n",
      " [ 1 22]]\n",
      "AUC: 0.9965217391304348\n"
     ]
    }
   ],
   "source": [
    "lr_w2v=LogisticRegression(solver = 'liblinear', C=10, penalty = 'l2')\n",
    "lr_w2v.fit(X_train_vectors_w2v, y_train)  #model\n",
    "\n",
    " \n",
    "y_predict = lr_w2v.predict(X_val_vectors_w2v)\n",
    "y_prob = lr_w2v.predict_proba(X_val_vectors_w2v)[:,1]\n",
    " \n",
    "\n",
    "print(classification_report(y_val,y_predict))\n",
    "print('Confusion Matrix:',confusion_matrix(y_val, y_predict))\n",
    " \n",
    "fpr, tpr, thresholds = roc_curve(y_val, y_prob)\n",
    "roc_auc = auc(fpr, tpr)\n",
    "print('AUC:', roc_auc)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "b0ae06be",
   "metadata": {},
   "outputs": [],
   "source": [
    "#naive bayes is best\n",
    "cheat_model=lr_tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "4e884f13",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def predict_sentence(sentence):\n",
    "    sentence = finalpreprocess(sentence)  \n",
    "    X_vector = tfidf_vectorizer.transform([sentence])  # convert the input sentence to a vector\n",
    "    y_predict = cheat_model.predict(X_vector)  # use the trained model to make a prediction\n",
    "    y_prob = cheat_model.predict_proba(X_vector)[:, 1]  # get the probability of the prediction\n",
    "    return y_predict, y_prob\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "49c7d943",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0]), array([0.44715957]))"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_sentence('how do i pass the test')\n",
    "# using 0.6 as the baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "97ac8b0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from data import save_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "7c9b64a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_model(cheat_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23578ad5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}