06_linear_regression__spam.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import io\n",
    "import time\n",
    "import urllib\n",
    "import zipfile\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import sklearn.metrics\n",
    "import sklearn.datasets\n",
    "import tensorflow as tf\n",
    "\n",
    "# Prints numpy arrays nicer\n",
    "np.set_printoptions(precision=2, suppress=True, linewidth=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download dataset.\n",
    "try:  # Prevent redownloading data if code block is repeatedly executed.\n",
    "    data\n",
    "except NameError:\n",
    "    data = []\n",
    "    url = ('https://archive.ics.uci.edu/'\n",
    "           + 'ml/machine-learning-databases/00228/smsspamcollection.zip')\n",
    "    with zipfile.ZipFile(\n",
    "            io.BytesIO(urllib.request.urlopen(url).read())) as archive:\n",
    "        # Uncomment the following lines for a description of the dataset.\n",
    "        # with archive.open('readme') as f:\n",
    "        #     print(f.read().decode('ISO-8859-1'))\n",
    "        with archive.open('SMSSpamCollection') as f:\n",
    "            for i, l in enumerate(f):\n",
    "                label, text = l.decode('ISO-8859-1').split(maxsplit=1)\n",
    "                words = tf.keras.preprocessing.text.text_to_word_sequence(text)\n",
    "                is_spam = label == 'spam'\n",
    "                data.append((words, is_spam))\n",
    "\n",
    "    # Build vocabulary (set of unique words) and assign word ids.\n",
    "    # Restrict vocabulary to 300 most frequent words and assign word ids.\n",
    "    vocab_size = 300\n",
    "    word_counter = collections.Counter()\n",
    "    for words, is_spam in data:\n",
    "        word_counter.update(words)\n",
    "\n",
    "    sorted_word_counts = sorted(word_counter.items(),\n",
    "                                key=lambda x: (-x[1], x[0]))[:vocab_size]\n",
    "    id_to_word = [word for word, count in sorted_word_counts]\n",
    "    word_to_id = {word: i for i, word in enumerate(id_to_word)}\n",
    "\n",
    "    # Transform lists of words into bag-of-words vectors.\n",
    "    for i in range(len(data)):\n",
    "        words, is_spam = data[i]\n",
    "        bag_of_words = np.zeros(shape=[vocab_size])\n",
    "        # if words:  # The preprocessing turns the SMS ':)' into an empty list.\n",
    "        for word in words:\n",
    "            if word in word_to_id:\n",
    "                bag_of_words[word_to_id[word]] += 1\n",
    "        data[i] = (bag_of_words, int(is_spam))\n",
    "\n",
    "# Perform 60% / 40% training/test split\n",
    "np.random.shuffle(data)\n",
    "split_index = int(len(data) * 0.6)\n",
    "train_data = data[:split_index]\n",
    "test_data = data[split_index:]\n",
    "print('Num training examples:', len(train_data))\n",
    "print('Num testing examples:', len(test_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperparameters\n",
    "learning_rate = 0.05\n",
    "num_epochs = 300\n",
    "num_features = len(train_data[0][0])\n",
    "batch_size = 100\n",
    "\n",
    "# Model Definition\n",
    "batch_x = tf.placeholder(tf.float32, shape=[None, num_features])\n",
    "batch_y = tf.placeholder(tf.int32, shape=[None])\n",
    "\n",
    "w = tf.Variable(tf.random_normal(shape=[num_features], mean=0, stddev=1))\n",
    "b = tf.Variable(0.0)\n",
    "\n",
    "score = tf.tensordot(batch_x, w, 1) + b\n",
    "y_prediction = tf.cast(tf.greater(score, 0.5), tf.int32)\n",
    "\n",
    "loss = tf.reduce_mean((tf.cast(batch_y, tf.float32) - score) ** 2)\n",
    "train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    # Training\n",
    "    time_before = time.time()\n",
    "    losses = []  # Storing losses so we can plot them later\n",
    "    for epoch in range(num_epochs):\n",
    "        np.random.shuffle(train_data)\n",
    "        cumulative_loss = 0\n",
    "        for i in range(0, len(train_data), batch_size):\n",
    "            _batch_x, _batch_y = zip(*train_data[i:i + batch_size])\n",
    "            _loss, _train_op = sess.run(\n",
    "                (loss, train_op),\n",
    "                feed_dict={batch_x: _batch_x, batch_y: _batch_y})\n",
    "            cumulative_loss += _loss * len(_batch_x)\n",
    "        average_loss = cumulative_loss / len(train_data)\n",
    "        if epoch % 15 == 14:\n",
    "            print('Epoch: {}, Loss: {}'.format(epoch + 1, average_loss))\n",
    "        losses.append(average_loss)\n",
    "    time_after = time.time()\n",
    "    print('Training took {:.2f}s.'.format(time_after - time_before))\n",
    "\n",
    "    # Introspection\n",
    "    _w, _b = sess.run([w, b])\n",
    "\n",
    "    # Prediction\n",
    "    train_xs, train_ys = zip(*train_data)\n",
    "    train_ys_prediction = sess.run(y_prediction, feed_dict={batch_x: train_xs})\n",
    "\n",
    "    test_xs, test_ys = zip(*test_data)\n",
    "    test_ys_prediction = sess.run(y_prediction, feed_dict={batch_x: test_xs})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(dpi=150)\n",
    "plt.title('Loss over Time')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Loss')\n",
    "plt.plot(range(len(losses)), losses, color='#458588')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Precision on Training data:',\n",
    "      sklearn.metrics.precision_score(train_ys, train_ys_prediction))\n",
    "print('Recall on Training data:',\n",
    "      sklearn.metrics.recall_score(train_ys, train_ys_prediction))\n",
    "print('F1-Score on Training data:',\n",
    "      sklearn.metrics.f1_score(train_ys, train_ys_prediction))\n",
    "print('Accuracy on Training data:',\n",
    "      sklearn.metrics.accuracy_score(train_ys, train_ys_prediction))\n",
    "print()\n",
    "print('Precision on Testing data:',\n",
    "      sklearn.metrics.precision_score(test_ys, test_ys_prediction))\n",
    "print('Recall on Testing data:',\n",
    "      sklearn.metrics.recall_score(test_ys, test_ys_prediction))\n",
    "print('F1-Score on Testing data:',\n",
    "      sklearn.metrics.f1_score(test_ys, test_ys_prediction))\n",
    "print('Accuracy on Testing data:',\n",
    "      sklearn.metrics.accuracy_score(test_ys, test_ys_prediction))\n",
    "\n",
    "train_num_spam = np.sum(train_ys)\n",
    "train_num_spam_prediction = np.sum(train_ys_prediction)\n",
    "train_num_ham = len(train_ys) - train_num_spam\n",
    "train_num_ham_prediction = len(train_ys_prediction) - train_num_spam_prediction\n",
    "test_num_spam = np.sum(test_ys)\n",
    "test_num_spam_prediction = np.sum(test_ys_prediction)\n",
    "test_num_ham = len(test_ys) - test_num_spam\n",
    "test_num_ham_prediction = len(test_ys_prediction) - test_num_spam_prediction\n",
    "\n",
    "plt.figure(dpi=150)\n",
    "plt.title('Class Distribution Actual vs Predicted: Training Data')\n",
    "plt.ylabel('Frequency')\n",
    "plt.bar([1, 2, 3.5, 4.5],\n",
    "        [train_num_spam, train_num_spam_prediction,\n",
    "         train_num_ham, train_num_ham_prediction],\n",
    "        tick_label=['Actual Spam', 'Predicted Spam',\n",
    "                    'Actual Ham', 'Predicted Ham'],\n",
    "        color=['#458588', '#CC241D', '#458588', '#CC241D'])\n",
    "plt.show()\n",
    "\n",
    "plt.figure(dpi=150)\n",
    "plt.title('Class Distribution Actual vs Predicted: Testing Data')\n",
    "plt.ylabel('Frequency')\n",
    "plt.bar([1, 2, 3.5, 4.5],\n",
    "        [test_num_spam, test_num_spam_prediction,\n",
    "         test_num_ham, test_num_ham_prediction],\n",
    "        tick_label=['Actual Spam', 'Predicted Spam',\n",
    "                    'Actual Ham', 'Predicted Ham'],\n",
    "        color=['#458588', '#CC241D'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_spammy_words = np.argsort(_w)[-50:][::-1]\n",
    "print('Most Spammy Words:')\n",
    "for i, word_id in enumerate(most_spammy_words):\n",
    "    print('{:2d}. {}'.format(i + 1, id_to_word[word_id]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}