final_pre_processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras_preprocessing.text import text_to_word_sequence, Tokenizer\n",
    "from nltk.tokenize import WordPunctTokenizer\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from collections import OrderedDict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## File path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path = 'C:/Users/SJH/OneDrive - korea.ac.kr/문서/MBTI 500.csv'\n",
    "USER_SAMPLE = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Date Load & check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(file_path, encoding = \"UTF-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "if USER_SAMPLE :\n",
    "    data = data.groupby('type').sample(frac = 0.2)\n",
    "data.reset_index(drop = True, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = data.drop('type', axis = 1), data['type']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Distribution of label and so on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 21213 entries, 0 to 21212\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   posts   21213 non-null  object\n",
      " 1   type    21213 non-null  object\n",
      "dtypes: object(2)\n",
      "memory usage: 331.6+ KB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "posts    0\n",
       "type     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "posts    21213\n",
      "dtype: int64 21213\n"
     ]
    }
   ],
   "source": [
    "print(X.nunique(), len(X)) # Same"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEGCAYAAACUzrmNAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWlUlEQVR4nO3df7DddX3n8eeLoJi2ssIkICZMw9iMK7CtSpaydTujYkts1bAuaLoiodKJS9Fqx7bC1pm6dtPFse6uoKC0dQnWls2iCHWNlU21tpYVLpoafsiQLYpZUhKxP8CxscH3/nE+qcebk/s9995z7o/c52PmzPmez/l+33mfH/e88v1xvidVhSRJUzlmvhuQJC18hoUkqZNhIUnqZFhIkjoZFpKkTsfOdwPjsmLFilqzZs18tyFJi8rdd9/9japaOXn8qA2LNWvWMDExMd9tSNKikuRrg8bdDCVJ6mRYSJI6GRaSpE6GhSSpk2EhSeo01rBI8tUku5LsTDLRxk5McnuSB9v1CX3zX5lkd5IHkpzXN35Wq7M7ydVJMs6+JUnfby7WLF5cVc+rqnXt9hXAjqpaC+xot0lyOrAROANYD1ybZFlb5jpgM7C2XdbPQd+SpGY+NkNtALa26a3A+X3jN1XVgap6CNgNnJ3kFOD4qrqjeudTv7FvGUnSHBh3WBTw6SR3J9ncxk6uqr0A7fqkNr4K+Hrfsnva2Ko2PXn8MEk2J5lIMrF///4RPgxJWtrG/Q3uF1bVI0lOAm5P8pUp5h20H6KmGD98sOp64HqAdevWFcD+635/eh0PsPKyi2ZdQ5IWs7GuWVTVI+16H3ALcDbwaNu0RLve12bfA5zat/hq4JE2vnrAuCRpjowtLJL8YJKnH5oGfhq4B7gN2NRm2wTc2qZvAzYmOS7JafR2ZN/ZNlU9nuScdhTUxX3LSJLmwDg3Q50M3NKOcj0W+IOq+lSSu4BtSS4FHgYuBKiqe5NsA+4DDgKXV9WTrdZlwA3AcmB7u0iS5sjYwqKq/gr4sQHjjwHnHmGZLcCWAeMTwJmj7lGSNBy/wS1J6mRYSJI6GRaSpE6GhSSpk2EhSepkWEiSOhkWkqROhoUkqZNhIUnqZFhIkjoZFpKkToaFJKmTYSFJ6mRYSJI6GRaSpE6GhSSp0zh/Ke+otffat81q+VN+8V0j6kSS5oZrFpKkToaFJKmTYSFJ6mRYSJI6GRaSpE6GhSSpk2EhSepkWEiSOhkWkqROhoUkqZNhIUnqZFhIkjoZFpKkToaFJKmTYSFJ6mRYSJI6jT0skixL8qUkn2i3T0xye5IH2/UJffNemWR3kgeSnNc3flaSXe2+q5Nk3H1Lkr5nLtYs3gzc33f7CmBHVa0FdrTbJDkd2AicAawHrk2yrC1zHbAZWNsu6+egb0lSM9awSLIa+Fngd/uGNwBb2/RW4Py+8Zuq6kBVPQTsBs5OcgpwfFXdUVUF3Ni3jCRpDox7zeK/Ab8GfLdv7OSq2gvQrk9q46uAr/fNt6eNrWrTk8cPk2RzkokkE/v37x/JA5AkjTEskrwc2FdVdw+7yICxmmL88MGq66tqXVWtW7ly5ZD/rCSpy7FjrP1C4JVJfgZ4GnB8kt8HHk1ySlXtbZuY9rX59wCn9i2/Gnikja8eMC5JmiNjW7OoqiuranVVraG34/pPquoi4DZgU5ttE3Brm74N2JjkuCSn0duRfWfbVPV4knPaUVAX9y0jSZoD41yzOJKrgG1JLgUeBi4EqKp7k2wD7gMOApdX1ZNtmcuAG4DlwPZ2kSTNkTkJi6r6LPDZNv0YcO4R5tsCbBkwPgGcOb4OJUlT8RvckqROhoUkqZNhIUnqZFhIkjoZFpKkToaFJKmTYSFJ6mRYSJI6GRaSpE6GhSSpk2EhSepkWEiSOhkWkqROhoUkqZNhIUnqZFhIkjoZFpKkToaFJKmTYSFJ6mRYSJI6GRaSpE6GhSSpk2EhSepkWEiSOhkWkqROhoUkqZNhIUnqZFhIkjoZFpKkToaFJKmTYSFJ6mRYSJI6jS0skjwtyZ1J/jLJvUn+Yxs/McntSR5s1yf0LXNlkt1JHkhyXt/4WUl2tfuuTpJx9S1JOtw41ywOAC+pqh8DngesT3IOcAWwo6rWAjvabZKcDmwEzgDWA9cmWdZqXQdsBta2y/ox9i1JmmRsYVE9T7SbT2mXAjYAW9v4VuD8Nr0BuKmqDlTVQ8Bu4OwkpwDHV9UdVVXAjX3LSJLmwFj3WSRZlmQnsA+4vaq+AJxcVXsB2vVJbfZVwNf7Ft/Txla16cnjg/69zUkmkkzs379/pI9FkpayY8dZvKqeBJ6X5BnALUnOnGL2QfshaorxQf/e9cD1AOvWrRs4z0J11wdfMavl/+Ub/mhEnUjS4ebkaKiq+lvgs/T2NTzaNi3Rrve12fYAp/Ytthp4pI2vHjAuSZoj4zwaamVboyDJcuClwFeA24BNbbZNwK1t+jZgY5LjkpxGb0f2nW1T1eNJzmlHQV3ct4wkaQ6MczPUKcDWdkTTMcC2qvpEkjuAbUkuBR4GLgSoqnuTbAPuAw4Cl7fNWACXATcAy4Ht7SJJmiNDhUWSHVV1btdYv6r6MvD8AeOPAQOXq6otwJYB4xPAVPs7JEljNGVYJHka8APAivbluUM7m48HnjXm3iRJC0TXmsUbgLfQC4a7+V5Y/D3w/vG1JUlaSKYMi6p6L/DeJG+qqmvmqCdJ0gIz1D6LqromyU8Aa/qXqaobx9SXJGkBGXYH94eBZwM7gUNHKB069YYk6Sg37KGz64DT27mZJElLzLBfyrsHeOY4G5EkLVzDrlmsAO5Lcie9U48DUFWvHEtXkqQFZdiweMc4m5AkLWzDHg31p+NuRJK0cA17NNTjfO+04E+l90NG36qq48fVmCRp4Rh2zeLp/beTnA+cPY6GJEkLz4xOUV5VHwdeMtpWJEkL1bCboV7Vd/MYet+78DsXkrREDHs0VP9vfh4EvgpsGHk3kqQFadh9Fj8/7kYkSQvXUPsskqxOckuSfUkeTfLRJKu7l5QkHQ2G3cH93+n9RvazgFXAH7UxSdISMOw+i5VV1R8ONyR5yxj60Qjd+qGXzWr5Da/3p84l9Qy7ZvGNJBclWdYuFwGPjbMxSdLCMWxYvB54NfDXwF7gAsCd3pK0RAy7Geo3gU1V9TcASU4EfpteiEiSjnLDhsWPHgoKgKr6ZpLnj6knLVAf/PB5s67xhtf98Qg6kTTXht0MdUySEw7daGsWwwaNJGmRG/YD/z3AXyS5md5pPl4NbBlbV5KkBWXYb3DfmGSC3skDA7yqqu4ba2eSpAVj6E1JLRwMCElagmZ0inJJ0tJiWEiSOhkWkqROhoUkqZNhIUnqZFhIkjqNLSySnJrkM0nuT3Jvkje38ROT3J7kwXbd/83wK5PsTvJAkvP6xs9Ksqvdd3WSjKtvSdLhxrlmcRB4a1U9FzgHuDzJ6cAVwI6qWgvsaLdp920EzgDWA9cmWdZqXQdsBta2y/ox9i1JmmRsYVFVe6vqi236ceB+er+ytwHY2mbbCpzfpjcAN1XVgap6CNgNnJ3kFOD4qrqjqgq4sW8ZSdIcmJN9FknWAM8HvgCcXFV7oRcowElttlXA1/sW29PGVrXpyeOD/p3NSSaSTOzfv3+kj0GSlrKxh0WSHwI+Crylqv5+qlkHjNUU44cPVl1fVeuqat3KlSun36wkaaCxhkWSp9ALio9U1cfa8KNt0xLtel8b3wOc2rf4auCRNr56wLgkaY6M82ioAL8H3F9V/6XvrtuATW16E3Br3/jGJMclOY3ejuw726aqx5Oc02pe3LeMJGkOjPMHjF4IvA7YlWRnG/sPwFXAtiSXAg8DFwJU1b1JttE7s+1B4PKqerItdxlwA7Ac2N4ukqQ5MrawqKo/Z/D+BoBzj7DMFgb8qFJVTQBnjq47SdJ0+A1uSVInw0KS1MmwkCR1MiwkSZ0MC0lSJ8NCktTJsJAkdTIsJEmdDAtJUifDQpLUybCQJHUyLCRJnQwLSVInw0KS1MmwkCR1MiwkSZ0MC0lSJ8NCktTJsJAkdTIsJEmdDAtJUifDQpLUybCQJHUyLCRJnQwLSVInw0KS1OnY+W5AS9vbbl4/6xrvuuBTI+hE0lRcs5AkdTIsJEmdDAtJUifDQpLUybCQJHUaW1gk+VCSfUnu6Rs7McntSR5s1yf03Xdlkt1JHkhyXt/4WUl2tfuuTpJx9SxJGmycaxY3AJOPi7wC2FFVa4Ed7TZJTgc2Ame0Za5Nsqwtcx2wGVjbLrM/1lKSNC1jC4uq+hzwzUnDG4CtbXorcH7f+E1VdaCqHgJ2A2cnOQU4vqruqKoCbuxbRpI0R+Z6n8XJVbUXoF2f1MZXAV/vm29PG1vVpiePD5Rkc5KJJBP79+8faeOStJQtlB3cg/ZD1BTjA1XV9VW1rqrWrVy5cmTNSdJSN9dh8WjbtES73tfG9wCn9s23Gnikja8eMC5JmkNzHRa3AZva9Cbg1r7xjUmOS3IavR3Zd7ZNVY8nOacdBXVx3zKSpDkythMJJvlD4EXAiiR7gN8ArgK2JbkUeBi4EKCq7k2yDbgPOAhcXlVPtlKX0TuyajmwvV0kSXNobGFRVT93hLvOPcL8W4AtA8YngDNH2JokaZoWyg5uSdICZlhIkjoZFpKkToaFJKmTYSFJ6mRYSJI6GRaSpE5j+56FNF9+5uNvnXWNT57/nhF0Ih09XLOQJHUyLCRJnQwLSVIn91lIQ/jZj10z6xr/61VvGkEn0vxwzUKS1MmwkCR1MiwkSZ0MC0lSJ8NCktTJsJAkdTIsJEmd/J6FNE9efvNHZl3jExe8dgSdSN1cs5AkdTIsJEmdDAtJUif3WUhHkQ03b591jVsveNkIOtHRxjULSVIn1ywkTemCj35x1jVu/rcvGEEnmk+uWUiSOhkWkqROhoUkqZP7LCTNuXfdsnfWNd72b04ZQScalmEh6aiw/X98Y9Y1XvaaFSPo5OjkZihJUifXLCTpCO79wKOzrnHGvz95BJ3Mv0UTFknWA+8FlgG/W1VXzXNLkjRtf/2er8xq+We+9Z+PqJPpWRRhkWQZ8H7gp4A9wF1Jbquq++a3M0maf/uu+cyslj/pTS/unGex7LM4G9hdVX9VVd8BbgI2zHNPkrRkpKrmu4dOSS4A1lfVL7TbrwN+vKreOGm+zcDmdvM5wANDlF8BzP4wCmtac3HUXAw9WnN+a/5wVa2cPLgoNkMBGTB2WMpV1fXA9dMqnExU1bqZNmZNay6mmouhR2suzJqLZTPUHuDUvturgUfmqRdJWnIWS1jcBaxNclqSpwIbgdvmuSdJWjIWxWaoqjqY5I3AH9M7dPZDVXXviMpPa7OVNa25yGsuhh6tuQBrLood3JKk+bVYNkNJkuaRYSFJ6lZVR+UFeKJdr6F3mO2b+u57H3AJvW+F7wTuA77dpncCFwA3AA+1218E/tUI614wk37bdH9fO4FfauNfBVaMsNYu4C/b8vf0zXNFm+ezwERf3XVt7Ly+eZ+g912XncCNwIuAv2s1/wHYO0zNNn1o2UO1/3cbfwfw/9rYPcC32/gxwNVtbBe9gyROA77Q5n0Y2N9Xb82kx/1p4JnAk33zjOLx1xHqvRz4Uvu37wPeAPx633z9ffzSgMc9rT6HeE5/ZcB7c6bP6YojvNcH1pv0HjxU6ydavUN/T/cBHwCOGVHdewb8TR7pOZ3Ja/XdYWsNeF/vBK7qez0faPN/fjo9DlF33ZSfqfP9oT5HYfEosBt46uQPzL557pm0/A20D3Xgp4Evj7ruTPqdYvmvMjgsZlvrO8DVA+b5LL0Ph5dN/hCaNM+6vtsvAj5B70P0B4EHgbOGqXlo2QF9vIP2wQY8l96H8THAzwE30z5M6B1ufULfcpcA75vicf8WvQ+cJ47w/prp4z84oNZT6B0KvrrdPg54zqD3R9fjnk6fwzynk95Ps3pOp1PvCMutof090Ts453PAq0ZZd6rnfJav1YHp1Jr8Ggx6T9H7AvK03k/D1D3SZalshtoP7AA2zXD5zwE/Moa6RzLKurOt9SSDHzvAu4G3z6RoVX0LuBt49ghr3k/vQ3MFcAqwt6q+2+7bU1V/M41yR3rNR9LrJE+n9+H3GEBVHaiqYc4+QJu//3EPMqo+Z/ucjqxeVR0E/oLBr9Go++w3q9dqxLU+x+BdCaPs8Z8slbAAuAp4azsp4XS9gt6q66jrTmWquu9OsrNd/sWYax0HnNU3z2v67rsDOJCk+yxk3295kl3Aq4B3TqPmT/b18euT70zy421yP7ANeEWb9z1Jnj/NHl9O7zVf3vdvjuLxL5tcr6q+Se97Q19L8odJXptk6L/N9rgDfHoGfU75nE4y2+d0uvU+0+77wuQFk/wAcC6D/y5nXHeSw177WbxWT5lBrV/um/+8ATVf0Z6LUdcdaFF8z2IUquqhJHcC/24ai707ydvpffhcOsK6nTrq/mpV3TzmWp9J8iS9NYvnVNXfHqH8f6L3v9a3DdnOT7br79DbR/KBadT8s6p6+YD5fznJRcDjwD9Ub716T5LnAC9plx1JLqyqHR39HXrcX249XF5Vz5ti/uk+/icH1auqX2hh/VLgV+idYfmSjlr9j/vbM+zzSM/pYapqps/pTOu9uKomn8/o2Ul20luTurWqto+o7iADn9MZvlb/OINa/7WqfntArY8k+Ta9TWoz6fFIdae0ZMKi+S162zI/N+T8w34oT7fusEZZd7q1XlxV30jyxBRBQVX9SZLfBM4Zsu6fAS+qqrNGWPOf3vxJnuircwDYDmxP8ihwPr1NclP5vg+SZNBpyWbV61S1dgG7knyY3oEHl3QsMvBxj7PPGT6no6z3fztCcSx9Dqg/3ddqlLVeW1UTMPXrPsoeYWlthqKqvkLvyICh/id1NNUdV4/NFuDXFlLNJC9I8qw2fQzwo8DXRtTbZLPt9YeSvKhv6HmMp9cF9ZyO6zUa52s/ytdqXK/7uOoutTUL6P3BfGke6x4LHJinujN57Mvbav8hn6qqK/pnqKpPJtk/zzUnOwn4nSTHtdt30jsSbLpG2euxtH0W/fVoH+JJPkjv0NBvMf3/BY66z0HvpZk8p1O9L0f1Gs227pF6POw5ZWav1bG0fRYjqDXOHqf8/PB0H3Oo/S/nLuDiGt25rUiyEthZVatGVVOjl+TNwKqqGvVa2EgluQX4nar65CzrLIr3ZZIN9DbtvHpM9UfyfI5LC9XdwJlV9XdHmm9JbYaaT221+B7g/4w4KF5Jbz/AlaOqqdFL8nv0DjB4/3z3MpX0jlL7Lr0vJc6mzqJ4XyZ5J/BO4D+Pqf5Ins9xSbKO3pfzrp0qKMA1C0nSEFyzkCR1MiwkSZ0MC0lSJ8NCGoEkz0jyi/PdhzQuhoU0Gs8ADAsdtQwLaTSuop23KMn/bMfuA5DkI0lemeSSJLcm+VSSB5L8Rt88FyW5sy3/wYz+xJTSrBgW0mhcwffOW/Q+4OcBkvwzej+wc+gLWWcDr6V3CoYLk6xL8lzgNcAL2/JPtnmkBWMpnu5DGquq+tMk709yEr3TsH+0qg62kxLeXlWPAST5GPCvgYPAWcBdbZ7lwL55aV46AsNCGo8P01s72Ai8vm988rdgi95vUWytqgX9bWctbW6GkkbjcXq/UHbIDcBbACad3uWnkpyYZDm902Z/nt6psy9oayK0+394DnqWhuaahTQCVfVYks8nuQfYXlW/muR+4OOTZv1zemsdPwL8Qd/vEryd3i/dHQP8I3A54zulujRtnhtKGoP0fvZzF/CCQydoS3IJsK6q3jifvUkz4WYoacSSvBT4CnBN15k8pcXCNQtJUifXLCRJnQwLSVInw0KS1MmwkCR1MiwkSZ3+PyxqMWUfZtjwAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      posts\n",
      "type       \n",
      "ENFJ    307\n",
      "ENFP   1233\n",
      "ENTJ    591\n",
      "ENTP   2345\n",
      "ESFJ     36\n",
      "ESFP     72\n",
      "ESTJ     96\n",
      "ESTP    397\n",
      "INFJ   2993\n",
      "INFP   2427\n",
      "INTJ   4485\n",
      "INTP   4992\n",
      "ISFJ    130\n",
      "ISFP    175\n",
      "ISTJ    249\n",
      "ISTP    685\n",
      "======================\n",
      "INTP    0.235327\n",
      "INTJ    0.211427\n",
      "INFJ    0.141093\n",
      "INFP    0.114411\n",
      "ENTP    0.110545\n",
      "ENFP    0.058125\n",
      "ISTP    0.032292\n",
      "ENTJ    0.027860\n",
      "ESTP    0.018715\n",
      "ENFJ    0.014472\n",
      "ISTJ    0.011738\n",
      "ISFP    0.008250\n",
      "ISFJ    0.006128\n",
      "ESTJ    0.004526\n",
      "ESFP    0.003394\n",
      "ESFJ    0.001697\n",
      "Name: type, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "sns.countplot(x='type', data=data, order=y.value_counts().index)\n",
    "plt.show()\n",
    "count = data.groupby('type').count()\n",
    "\n",
    "count.sort_values(by=['posts'], ascending=False)\n",
    "print(count, y.value_counts(normalize=True), sep = '\\n======================\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Top 3000 words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: '<OOV>',\n",
       " 2: 'like',\n",
       " 3: 'think',\n",
       " 4: 'people',\n",
       " 5: 'get',\n",
       " 6: 'thing',\n",
       " 7: 'make',\n",
       " 8: 'know',\n",
       " 9: 'feel',\n",
       " 10: 'one',\n",
       " 11: 'go',\n",
       " 12: 'time',\n",
       " 13: 'say',\n",
       " 14: 'good',\n",
       " 15: 'would',\n",
       " 16: 'really',\n",
       " 17: 'want',\n",
       " 18: 'way',\n",
       " 19: 'see',\n",
       " 20: 'also',\n",
       " 21: 'work',\n",
       " 22: 'much',\n",
       " 23: 'something',\n",
       " 24: 'try',\n",
       " 25: 'take',\n",
       " 26: 'find',\n",
       " 27: 'type',\n",
       " 28: 'even',\n",
       " 29: 'well',\n",
       " 30: 'friend',\n",
       " 31: 'need',\n",
       " 32: 'lot',\n",
       " 33: 'use',\n",
       " 34: 'love',\n",
       " 35: 'life',\n",
       " 36: 'someone',\n",
       " 37: 'mean',\n",
       " 38: 'come',\n",
       " 39: 'look',\n",
       " 40: 'could',\n",
       " 41: 'talk',\n",
       " 42: 'though',\n",
       " 43: 'person',\n",
       " 44: 'give',\n",
       " 45: 'always',\n",
       " 46: 'never',\n",
       " 47: 'right',\n",
       " 48: 'interest',\n",
       " 49: 'seem',\n",
       " 50: 'actually',\n",
       " 51: 'understand',\n",
       " 52: 'year',\n",
       " 53: 'help',\n",
       " 54: 'still',\n",
       " 55: 'tell',\n",
       " 56: 'u',\n",
       " 57: 'read',\n",
       " 58: 'point',\n",
       " 59: 'pretty',\n",
       " 60: 'function',\n",
       " 61: 'day',\n",
       " 62: 'sure',\n",
       " 63: 'start',\n",
       " 64: 'bad',\n",
       " 65: 'first',\n",
       " 66: 'probably',\n",
       " 67: 'long',\n",
       " 68: 'intj',\n",
       " 69: 'intp',\n",
       " 70: 'different',\n",
       " 71: 'anything',\n",
       " 72: 'best',\n",
       " 73: 'relationship',\n",
       " 74: 'idea',\n",
       " 75: 'around',\n",
       " 76: 'might',\n",
       " 77: 'learn',\n",
       " 78: 'world',\n",
       " 79: 'many',\n",
       " 80: 'others',\n",
       " 81: 'fe',\n",
       " 82: 'maybe',\n",
       " 83: 'ask',\n",
       " 84: 'back',\n",
       " 85: 'experience',\n",
       " 86: 'kind',\n",
       " 87: 'hard',\n",
       " 88: 'problem',\n",
       " 89: 'live',\n",
       " 90: 'mind',\n",
       " 91: 'keep',\n",
       " 92: 'sometimes',\n",
       " 93: 'post',\n",
       " 94: 'end',\n",
       " 95: 'reason',\n",
       " 96: 'may',\n",
       " 97: 'great',\n",
       " 98: 'thank',\n",
       " 99: 'question',\n",
       " 100: 'let',\n",
       " 101: 'care',\n",
       " 102: 'happen',\n",
       " 103: 'yes',\n",
       " 104: 'change',\n",
       " 105: 'every',\n",
       " 106: 'everything',\n",
       " 107: 'little',\n",
       " 108: 'part',\n",
       " 109: 'everyone',\n",
       " 110: 'sound',\n",
       " 111: 'infj',\n",
       " 112: 'two',\n",
       " 113: 'sense',\n",
       " 114: 'put',\n",
       " 115: 'yeah',\n",
       " 116: 'usually',\n",
       " 117: 'enough',\n",
       " 118: 'often',\n",
       " 119: 'new',\n",
       " 120: 'bite',\n",
       " 121: 'since',\n",
       " 122: 'guy',\n",
       " 123: 'write',\n",
       " 124: 'play',\n",
       " 125: 'believe',\n",
       " 126: 'social',\n",
       " 127: 'else',\n",
       " 128: 'without',\n",
       " 129: 'self',\n",
       " 130: 'ever',\n",
       " 131: 'become',\n",
       " 132: 'high',\n",
       " 133: 'fuck',\n",
       " 134: 'school',\n",
       " 135: 'stuff',\n",
       " 136: 'infp',\n",
       " 137: 'guess',\n",
       " 138: 'another',\n",
       " 139: 'word',\n",
       " 140: 'le',\n",
       " 141: 'least',\n",
       " 142: 'fi',\n",
       " 143: 'show',\n",
       " 144: 'wrong',\n",
       " 145: 'game',\n",
       " 146: 'tend',\n",
       " 147: 'job',\n",
       " 148: 'situation',\n",
       " 149: 'nothing',\n",
       " 150: 'mbti',\n",
       " 151: 'place',\n",
       " 152: 'definitely',\n",
       " 153: 'big',\n",
       " 154: 'agree',\n",
       " 155: 'call',\n",
       " 156: 'test',\n",
       " 157: 'enjoy',\n",
       " 158: 'meet',\n",
       " 159: 'shit',\n",
       " 160: 'close',\n",
       " 161: 'lol',\n",
       " 162: 'either',\n",
       " 163: 'rather',\n",
       " 164: 'entp',\n",
       " 165: 'emotion',\n",
       " 166: 'fact',\n",
       " 167: 'leave',\n",
       " 168: 'ni',\n",
       " 169: 'easy',\n",
       " 170: 'personality',\n",
       " 171: 'answer',\n",
       " 172: 'introvert',\n",
       " 173: 'etc',\n",
       " 174: 'true',\n",
       " 175: 'ti',\n",
       " 176: 'real',\n",
       " 177: 'anyone',\n",
       " 178: 'able',\n",
       " 179: 'win',\n",
       " 180: 'quite',\n",
       " 181: 'enfp',\n",
       " 182: 'ne',\n",
       " 183: 'consider',\n",
       " 184: 'last',\n",
       " 185: 'base',\n",
       " 186: 'value',\n",
       " 187: 'example',\n",
       " 188: 'hear',\n",
       " 189: 'fun',\n",
       " 190: 'old',\n",
       " 191: 'stop',\n",
       " 192: 'far',\n",
       " 193: 'happy',\n",
       " 194: 'woman',\n",
       " 195: 'oh',\n",
       " 196: 'however',\n",
       " 197: 'matter',\n",
       " 198: 'move',\n",
       " 199: 'especially',\n",
       " 200: 'human',\n",
       " 201: 'away',\n",
       " 202: 'issue',\n",
       " 203: 'relate',\n",
       " 204: 'watch',\n",
       " 205: 'conversation',\n",
       " 206: 'date',\n",
       " 207: 'sort',\n",
       " 208: 'book',\n",
       " 209: 'nice',\n",
       " 210: 'realize',\n",
       " 211: 'spend',\n",
       " 212: 'head',\n",
       " 213: 'group',\n",
       " 214: 'focus',\n",
       " 215: 'emotional',\n",
       " 216: 'almost',\n",
       " 217: 'alone',\n",
       " 218: 'open',\n",
       " 219: 'already',\n",
       " 220: 'hate',\n",
       " 221: 'remember',\n",
       " 222: 'important',\n",
       " 223: 'share',\n",
       " 224: 'girl',\n",
       " 225: 'likely',\n",
       " 226: 'te',\n",
       " 227: 'explain',\n",
       " 228: 'plan',\n",
       " 229: 'listen',\n",
       " 230: 'whole',\n",
       " 231: 'case',\n",
       " 232: 'whatever',\n",
       " 233: 'man',\n",
       " 234: 'deal',\n",
       " 235: 'break',\n",
       " 236: 'music',\n",
       " 237: 'act',\n",
       " 238: 'speak',\n",
       " 239: 'side',\n",
       " 240: 'completely',\n",
       " 241: 'instead',\n",
       " 242: 'exactly',\n",
       " 243: 'figure',\n",
       " 244: 'turn',\n",
       " 245: 'se',\n",
       " 246: 'family',\n",
       " 247: 'certain',\n",
       " 248: 'late',\n",
       " 249: 'personal',\n",
       " 250: 'lose',\n",
       " 251: 'si',\n",
       " 252: 'opinion',\n",
       " 253: 'system',\n",
       " 254: 'yet',\n",
       " 255: 'process',\n",
       " 256: 'course',\n",
       " 257: 'similar',\n",
       " 258: 'together',\n",
       " 259: 'level',\n",
       " 260: 'small',\n",
       " 261: 'intjs',\n",
       " 262: 'kid',\n",
       " 263: 'haha',\n",
       " 264: 'general',\n",
       " 265: 'term',\n",
       " 266: 'r',\n",
       " 267: 'run',\n",
       " 268: 'strong',\n",
       " 269: 'cause',\n",
       " 270: 'intps',\n",
       " 271: 'thought',\n",
       " 272: 'comment',\n",
       " 273: 'possible',\n",
       " 274: 'god',\n",
       " 275: 'month',\n",
       " 276: 'state',\n",
       " 277: 'week',\n",
       " 278: 'information',\n",
       " 279: 'past',\n",
       " 280: 'im',\n",
       " 281: 'hope',\n",
       " 282: 'sorry',\n",
       " 283: 'view',\n",
       " 284: 'set',\n",
       " 285: 'edit',\n",
       " 286: 'hour',\n",
       " 287: 'hand',\n",
       " 288: 'control',\n",
       " 289: 'mostly',\n",
       " 290: 'moment',\n",
       " 291: 'describe',\n",
       " 292: 'basically',\n",
       " 293: 'fit',\n",
       " 294: 'study',\n",
       " 295: 'bring',\n",
       " 296: 'grow',\n",
       " 297: 'follow',\n",
       " 298: 'lead',\n",
       " 299: 'develop',\n",
       " 300: 'story',\n",
       " 301: 'second',\n",
       " 302: 'next',\n",
       " 303: 'deep',\n",
       " 304: 'weird',\n",
       " 305: 'theory',\n",
       " 306: 'money',\n",
       " 307: 'cool',\n",
       " 308: 'along',\n",
       " 309: 'depend',\n",
       " 310: 'notice',\n",
       " 311: 'create',\n",
       " 312: 'difference',\n",
       " 313: 'personally',\n",
       " 314: 'advice',\n",
       " 315: 'brain',\n",
       " 316: 'class',\n",
       " 317: 'whether',\n",
       " 318: 'simply',\n",
       " 319: 'decide',\n",
       " 320: 'thread',\n",
       " 321: 'entj',\n",
       " 322: 'wish',\n",
       " 323: 'unless',\n",
       " 324: 'stay',\n",
       " 325: 'home',\n",
       " 326: 'e',\n",
       " 327: 'order',\n",
       " 328: 'face',\n",
       " 329: 'generally',\n",
       " 330: 'exist',\n",
       " 331: 'drive',\n",
       " 332: 'child',\n",
       " 333: 'free',\n",
       " 334: 'college',\n",
       " 335: 'young',\n",
       " 336: 'video',\n",
       " 337: 'choose',\n",
       " 338: 'pay',\n",
       " 339: 'argument',\n",
       " 340: 'must',\n",
       " 341: 'hold',\n",
       " 342: 'prefer',\n",
       " 343: 'ago',\n",
       " 344: 'common',\n",
       " 345: 'honestly',\n",
       " 346: 'bore',\n",
       " 347: 'goal',\n",
       " 348: 'action',\n",
       " 349: 'expect',\n",
       " 350: 'lack',\n",
       " 351: 'pick',\n",
       " 352: 'parent',\n",
       " 353: 'hurt',\n",
       " 354: 'assume',\n",
       " 355: 'decision',\n",
       " 356: 'future',\n",
       " 357: 'accept',\n",
       " 358: 'istp',\n",
       " 359: 'movie',\n",
       " 360: 'anyway',\n",
       " 361: 'infjs',\n",
       " 362: 'drink',\n",
       " 363: 'eye',\n",
       " 364: 'ahough',\n",
       " 365: 'add',\n",
       " 366: 'worth',\n",
       " 367: 'trust',\n",
       " 368: 'force',\n",
       " 369: 'form',\n",
       " 370: 'sit',\n",
       " 371: 'reality',\n",
       " 372: 'perspective',\n",
       " 373: 'totally',\n",
       " 374: 'topic',\n",
       " 375: 'age',\n",
       " 376: 'okay',\n",
       " 377: 'eat',\n",
       " 378: 'logic',\n",
       " 379: 'wonder',\n",
       " 380: 'dream',\n",
       " 381: 'mention',\n",
       " 382: 'language',\n",
       " 383: 'low',\n",
       " 384: 'couple',\n",
       " 385: 'towards',\n",
       " 386: 'worry',\n",
       " 387: 'walk',\n",
       " 388: 'stick',\n",
       " 389: 'party',\n",
       " 390: 'night',\n",
       " 391: 'wait',\n",
       " 392: 'outside',\n",
       " 393: 'fall',\n",
       " 394: 'society',\n",
       " 395: 'allow',\n",
       " 396: 'line',\n",
       " 397: 'attention',\n",
       " 398: 'character',\n",
       " 399: 'check',\n",
       " 400: 'name',\n",
       " 401: 'truth',\n",
       " 402: 'user',\n",
       " 403: 'full',\n",
       " 404: 'fine',\n",
       " 405: 'super',\n",
       " 406: 'forget',\n",
       " 407: 'imagine',\n",
       " 408: 'rule',\n",
       " 409: 'stand',\n",
       " 410: 'please',\n",
       " 411: 'suppose',\n",
       " 412: 'stupid',\n",
       " 413: 'men',\n",
       " 414: 'build',\n",
       " 415: 'appreciate',\n",
       " 416: 'response',\n",
       " 417: 'judge',\n",
       " 418: 'sex',\n",
       " 419: 'list',\n",
       " 420: 'skill',\n",
       " 421: 'difficu',\n",
       " 422: 'science',\n",
       " 423: 'favorite',\n",
       " 424: 'individual',\n",
       " 425: 'joke',\n",
       " 426: 'logical',\n",
       " 427: 'energy',\n",
       " 428: 'istj',\n",
       " 429: 'ok',\n",
       " 430: 'absolutely',\n",
       " 431: 'awesome',\n",
       " 432: 'easily',\n",
       " 433: 'annoy',\n",
       " 434: 'infps',\n",
       " 435: 'female',\n",
       " 436: 'house',\n",
       " 437: 'funny',\n",
       " 438: 'power',\n",
       " 439: 'sleep',\n",
       " 440: 'choice',\n",
       " 441: 'support',\n",
       " 442: 'behavior',\n",
       " 443: 'avoid',\n",
       " 444: 'respect',\n",
       " 445: 'die',\n",
       " 446: 'seriously',\n",
       " 447: 'perhaps',\n",
       " 448: 'extremely',\n",
       " 449: 'clear',\n",
       " 450: 'short',\n",
       " 451: 'smart',\n",
       " 452: 'step',\n",
       " 453: 'body',\n",
       " 454: 'detail',\n",
       " 455: 'specific',\n",
       " 456: 'hang',\n",
       " 457: 'effort',\n",
       " 458: 'teach',\n",
       " 459: 'room',\n",
       " 460: 'due',\n",
       " 461: 'express',\n",
       " 462: 'op',\n",
       " 463: 'anxiety',\n",
       " 464: 'literally',\n",
       " 465: 'comfortable',\n",
       " 466: 'single',\n",
       " 467: 'knowledge',\n",
       " 468: 'ability',\n",
       " 469: 'stress',\n",
       " 470: 'normal',\n",
       " 471: 'art',\n",
       " 472: 'sub',\n",
       " 473: 'project',\n",
       " 474: 'nature',\n",
       " 475: 'miss',\n",
       " 476: 'perfect',\n",
       " 477: 'number',\n",
       " 478: 'hell',\n",
       " 479: 'mine',\n",
       " 480: 'picture',\n",
       " 481: 'partner',\n",
       " 482: 'buy',\n",
       " 483: 'non',\n",
       " 484: 'description',\n",
       " 485: 'physical',\n",
       " 486: 'song',\n",
       " 487: 'honest',\n",
       " 488: 'lie',\n",
       " 489: 'half',\n",
       " 490: 'suck',\n",
       " 491: 'p',\n",
       " 492: 'male',\n",
       " 493: 'amp',\n",
       " 494: 'kinda',\n",
       " 495: 'amount',\n",
       " 496: 'space',\n",
       " 497: 'v',\n",
       " 498: 'connection',\n",
       " 499: 'chance',\n",
       " 500: 'apply',\n",
       " 501: 'internet',\n",
       " 502: 'rest',\n",
       " 503: 'w',\n",
       " 504: 'mom',\n",
       " 505: 'early',\n",
       " 506: 'negative',\n",
       " 507: 'include',\n",
       " 508: 'cognitive',\n",
       " 509: 'begin',\n",
       " 510: 'trait',\n",
       " 511: 'mental',\n",
       " 512: 'require',\n",
       " 513: 'draw',\n",
       " 514: 'top',\n",
       " 515: 'research',\n",
       " 516: 'aware',\n",
       " 517: 'argue',\n",
       " 518: 'approach',\n",
       " 519: 'concept',\n",
       " 520: 'struggle',\n",
       " 521: 'n',\n",
       " 522: 'fear',\n",
       " 523: 'note',\n",
       " 524: 'within',\n",
       " 525: 'country',\n",
       " 526: 'involve',\n",
       " 527: 'fight',\n",
       " 528: 'entps',\n",
       " 529: 'bother',\n",
       " 530: 'kill',\n",
       " 531: 'minute',\n",
       " 532: 'enfj',\n",
       " 533: 'present',\n",
       " 534: 'degree',\n",
       " 535: 'online',\n",
       " 536: 'hit',\n",
       " 537: 'opposite',\n",
       " 538: 'subject',\n",
       " 539: 'otherwise',\n",
       " 540: 'simple',\n",
       " 541: 'link',\n",
       " 542: 'except',\n",
       " 543: 'connect',\n",
       " 544: 'food',\n",
       " 545: 'cannot',\n",
       " 546: 'x',\n",
       " 547: 'large',\n",
       " 548: 'isfj',\n",
       " 549: 'hey',\n",
       " 550: 'depression',\n",
       " 551: 'religion',\n",
       " 552: 'suggest',\n",
       " 553: 'today',\n",
       " 554: 'f',\n",
       " 555: 'friendship',\n",
       " 556: 'crazy',\n",
       " 557: 'correct',\n",
       " 558: 'sad',\n",
       " 559: 'amaze',\n",
       " 560: 'recently',\n",
       " 561: 'ignore',\n",
       " 562: 'dominant',\n",
       " 563: 'truly',\n",
       " 564: 'laugh',\n",
       " 565: 'belief',\n",
       " 566: 'huge',\n",
       " 567: 'cry',\n",
       " 568: 'push',\n",
       " 569: 'isfp',\n",
       " 570: 'constantly',\n",
       " 571: 'eventually',\n",
       " 572: 'fail',\n",
       " 573: 'interaction',\n",
       " 574: 'obviously',\n",
       " 575: 'positive',\n",
       " 576: 'extroverted',\n",
       " 577: 'serious',\n",
       " 578: 'actual',\n",
       " 579: 'heart',\n",
       " 580: 'estp',\n",
       " 581: 'perceive',\n",
       " 582: 'practice',\n",
       " 583: 'surprise',\n",
       " 584: 'area',\n",
       " 585: 'light',\n",
       " 586: 'discussion',\n",
       " 587: 'enfps',\n",
       " 588: 'throw',\n",
       " 589: 'necessarily',\n",
       " 590: 'saw',\n",
       " 591: 'doubt',\n",
       " 592: 'certainly',\n",
       " 593: 'desire',\n",
       " 594: 'dad',\n",
       " 595: 'reddit',\n",
       " 596: 'major',\n",
       " 597: 'mother',\n",
       " 598: 'event',\n",
       " 599: 'fix',\n",
       " 600: 'three',\n",
       " 601: 'anymore',\n",
       " 602: 'random',\n",
       " 603: 'effect',\n",
       " 604: 'position',\n",
       " 605: 'compare',\n",
       " 606: 'career',\n",
       " 607: 'solve',\n",
       " 608: 'data',\n",
       " 609: 'company',\n",
       " 610: 'resus',\n",
       " 611: 'provide',\n",
       " 612: 'prove',\n",
       " 613: 'environment',\n",
       " 614: 'behind',\n",
       " 615: 'program',\n",
       " 616: 'option',\n",
       " 617: 'inferior',\n",
       " 618: 'business',\n",
       " 619: 'useful',\n",
       " 620: 'fast',\n",
       " 621: 'debate',\n",
       " 622: 'confuse',\n",
       " 623: 'inside',\n",
       " 624: 'damn',\n",
       " 625: 'emotionally',\n",
       " 626: 'treat',\n",
       " 627: 'concern',\n",
       " 628: 'extrovert',\n",
       " 629: 'possibility',\n",
       " 630: 'heahy',\n",
       " 631: 'natural',\n",
       " 632: 'death',\n",
       " 633: 'quality',\n",
       " 634: 'entire',\n",
       " 635: 'black',\n",
       " 636: 'wear',\n",
       " 637: 'across',\n",
       " 638: 'particular',\n",
       " 639: 'luck',\n",
       " 640: 'improve',\n",
       " 641: 'cut',\n",
       " 642: 'pattern',\n",
       " 643: 'touch',\n",
       " 644: 'reply',\n",
       " 645: 'finish',\n",
       " 646: 'identify',\n",
       " 647: 'conclusion',\n",
       " 648: 'currently',\n",
       " 649: 'mistake',\n",
       " 650: 'math',\n",
       " 651: 'resu',\n",
       " 652: 'esfp',\n",
       " 653: 'purpose',\n",
       " 654: 'seek',\n",
       " 655: 'claim',\n",
       " 656: 'history',\n",
       " 657: 'accurate',\n",
       " 658: 'current',\n",
       " 659: 'computer',\n",
       " 660: 'hat',\n",
       " 661: 'limit',\n",
       " 662: 'b',\n",
       " 663: 'pas',\n",
       " 664: 'offer',\n",
       " 665: 'brother',\n",
       " 666: 'thats',\n",
       " 667: 'gonna',\n",
       " 668: 'quickly',\n",
       " 669: 'respond',\n",
       " 670: 'continue',\n",
       " 671: 'define',\n",
       " 672: 'challenge',\n",
       " 673: 'intuition',\n",
       " 674: 'wow',\n",
       " 675: 'memory',\n",
       " 676: 'glad',\n",
       " 677: 'car',\n",
       " 678: 'phone',\n",
       " 679: 'complete',\n",
       " 680: 'regard',\n",
       " 681: 'save',\n",
       " 682: 'affect',\n",
       " 683: 'disagree',\n",
       " 684: 'handle',\n",
       " 685: 'main',\n",
       " 686: 'waste',\n",
       " 687: 'esfj',\n",
       " 688: 'solution',\n",
       " 689: 'evidence',\n",
       " 690: 'depress',\n",
       " 691: 'send',\n",
       " 692: 'drug',\n",
       " 693: 'law',\n",
       " 694: 'moral',\n",
       " 695: 'j',\n",
       " 696: 'naturally',\n",
       " 697: 'trouble',\n",
       " 698: 'estj',\n",
       " 699: 'middle',\n",
       " 700: 'clearly',\n",
       " 701: 'text',\n",
       " 702: 'style',\n",
       " 703: 'quiet',\n",
       " 704: 'angry',\n",
       " 705: 'source',\n",
       " 706: 'reach',\n",
       " 707: 'interact',\n",
       " 708: 'aspect',\n",
       " 709: 'romantic',\n",
       " 710: 'field',\n",
       " 711: 'dark',\n",
       " 712: 'stereotype',\n",
       " 713: 'recommend',\n",
       " 714: 'remind',\n",
       " 715: 'catch',\n",
       " 716: 'piece',\n",
       " 717: 'balance',\n",
       " 718: 'discus',\n",
       " 719: 'awkward',\n",
       " 720: 'marry',\n",
       " 721: 'several',\n",
       " 722: 'universe',\n",
       " 723: 'average',\n",
       " 724: 'match',\n",
       " 725: 'finally',\n",
       " 726: 'model',\n",
       " 727: 'dude',\n",
       " 728: 'cold',\n",
       " 729: 'dom',\n",
       " 730: 'engage',\n",
       " 731: 'benefit',\n",
       " 732: 'contact',\n",
       " 733: 'white',\n",
       " 734: 'attempt',\n",
       " 735: 'fair',\n",
       " 736: 'curious',\n",
       " 737: 'dog',\n",
       " 738: 'obvious',\n",
       " 739: 'highly',\n",
       " 740: 'context',\n",
       " 741: 'philosophy',\n",
       " 742: 'conflict',\n",
       " 743: 'teacher',\n",
       " 744: 'soon',\n",
       " 745: 'activity',\n",
       " 746: 'definition',\n",
       " 747: 'preference',\n",
       " 748: 'bear',\n",
       " 749: 'role',\n",
       " 750: 'somewhere',\n",
       " 751: 'welcome',\n",
       " 752: 'train',\n",
       " 753: 'somewhat',\n",
       " 754: 'special',\n",
       " 755: 'bullshit',\n",
       " 756: 'pain',\n",
       " 757: 'community',\n",
       " 758: 'attract',\n",
       " 759: 'message',\n",
       " 760: 'public',\n",
       " 761: 'sign',\n",
       " 762: 'hobby',\n",
       " 763: 'intelligence',\n",
       " 764: 'beautiful',\n",
       " 765: 'fairly',\n",
       " 766: 'afraid',\n",
       " 767: 'standard',\n",
       " 768: 'statement',\n",
       " 769: 'communication',\n",
       " 770: 'entirely',\n",
       " 771: 'helpful',\n",
       " 772: 'strange',\n",
       " 773: 'pull',\n",
       " 774: 'hide',\n",
       " 775: 'account',\n",
       " 776: 'communicate',\n",
       " 777: 'manage',\n",
       " 778: 'basic',\n",
       " 779: 'potential',\n",
       " 780: 'upset',\n",
       " 781: 'reaction',\n",
       " 782: 'frustrate',\n",
       " 783: 'straight',\n",
       " 784: 'particularly',\n",
       " 785: 'sister',\n",
       " 786: 'article',\n",
       " 787: 'animal',\n",
       " 788: 'direct',\n",
       " 789: 'student',\n",
       " 790: 'tire',\n",
       " 791: 'city',\n",
       " 792: 'c',\n",
       " 793: 'objective',\n",
       " 794: 'th',\n",
       " 795: 'somehow',\n",
       " 796: 'rarely',\n",
       " 797: 'rational',\n",
       " 798: 'creative',\n",
       " 799: 'dislike',\n",
       " 800: 'boyfriend',\n",
       " 801: 'ex',\n",
       " 802: 'front',\n",
       " 803: 'excite',\n",
       " 804: 'convince',\n",
       " 805: 'opportunity',\n",
       " 806: 'influence',\n",
       " 807: 'bias',\n",
       " 808: 'vote',\n",
       " 809: 'english',\n",
       " 810: 'possibly',\n",
       " 811: 'nobody',\n",
       " 812: 'whenever',\n",
       " 813: 'smile',\n",
       " 814: 'fully',\n",
       " 815: 'plus',\n",
       " 816: 'cuure',\n",
       " 817: 'raise',\n",
       " 818: 'bunch',\n",
       " 819: 'appear',\n",
       " 820: 'quote',\n",
       " 821: 'task',\n",
       " 822: 'subjective',\n",
       " 823: 'mood',\n",
       " 824: 'attack',\n",
       " 825: 'admit',\n",
       " 826: 'spot',\n",
       " 827: 'voice',\n",
       " 828: 'forum',\n",
       " 829: 'despite',\n",
       " 830: 'terrible',\n",
       " 831: 'government',\n",
       " 832: 'ideal',\n",
       " 833: 'asshole',\n",
       " 834: 'war',\n",
       " 835: 'analyze',\n",
       " 836: 'team',\n",
       " 837: 'drop',\n",
       " 838: 'recognize',\n",
       " 839: 'motivation',\n",
       " 840: 'exact',\n",
       " 841: 'rare',\n",
       " 842: 'confidence',\n",
       " 843: 'tendency',\n",
       " 844: 'medium',\n",
       " 845: 'incredibly',\n",
       " 846: 'gender',\n",
       " 847: 'comfort',\n",
       " 848: 'none',\n",
       " 849: 'determine',\n",
       " 850: 'subreddit',\n",
       " 851: 'clean',\n",
       " 852: 'path',\n",
       " 853: 'girlfriend',\n",
       " 854: 'thus',\n",
       " 855: 'version',\n",
       " 856: 'father',\n",
       " 857: 'complex',\n",
       " 858: 'cat',\n",
       " 859: 'design',\n",
       " 860: 'count',\n",
       " 861: 'factor',\n",
       " 862: 'lazy',\n",
       " 863: 'star',\n",
       " 864: 'grade',\n",
       " 865: 'explore',\n",
       " 866: 'score',\n",
       " 867: 'page',\n",
       " 868: 'forward',\n",
       " 869: 'regardless',\n",
       " 870: 'suffer',\n",
       " 871: 'mess',\n",
       " 872: 'water',\n",
       " 873: 'tool',\n",
       " 874: 'religious',\n",
       " 875: 'hop',\n",
       " 876: 'intelligent',\n",
       " 877: 'period',\n",
       " 878: 'fill',\n",
       " 879: 'gain',\n",
       " 880: 'sport',\n",
       " 881: 'gift',\n",
       " 882: 'door',\n",
       " 883: 'engineer',\n",
       " 884: 'quick',\n",
       " 885: 'therefore',\n",
       " 886: 'upon',\n",
       " 887: 'intuitive',\n",
       " 888: 'perception',\n",
       " 889: 'shut',\n",
       " 890: 'search',\n",
       " 891: 'jump',\n",
       " 892: 'uncomfortable',\n",
       " 893: 'specifically',\n",
       " 894: 'structure',\n",
       " 895: 'content',\n",
       " 896: 'ton',\n",
       " 897: 'majority',\n",
       " 898: 'letter',\n",
       " 899: 'empathy',\n",
       " 900: 'shoot',\n",
       " 901: 'beyond',\n",
       " 902: 'xd',\n",
       " 903: 'heah',\n",
       " 904: 'dead',\n",
       " 905: 'necessary',\n",
       " 906: 'unfortunately',\n",
       " 907: 'exercise',\n",
       " 908: 'calm',\n",
       " 909: 'happiness',\n",
       " 910: 'key',\n",
       " 911: 'travel',\n",
       " 912: 'confident',\n",
       " 913: 'muiple',\n",
       " 914: 'sensitive',\n",
       " 915: 'socially',\n",
       " 916: 'color',\n",
       " 917: 'fake',\n",
       " 918: 'mix',\n",
       " 919: 'achieve',\n",
       " 920: 'hug',\n",
       " 921: 'psychology',\n",
       " 922: 'slow',\n",
       " 923: 'overall',\n",
       " 924: 'sensor',\n",
       " 925: 'apparently',\n",
       " 926: 'observe',\n",
       " 927: 'impossible',\n",
       " 928: 'flaw',\n",
       " 929: 'internal',\n",
       " 930: 'rock',\n",
       " 931: 'bed',\n",
       " 932: 'risk',\n",
       " 933: 'tv',\n",
       " 934: 'original',\n",
       " 935: 'method',\n",
       " 936: 'dumb',\n",
       " 937: 'relax',\n",
       " 938: 'poor',\n",
       " 939: 'four',\n",
       " 940: 'perfectly',\n",
       " 941: 'attitude',\n",
       " 942: 'plenty',\n",
       " 943: 'beat',\n",
       " 944: 'insight',\n",
       " 945: 'political',\n",
       " 946: 'inner',\n",
       " 947: 'stack',\n",
       " 948: 'wall',\n",
       " 949: 'image',\n",
       " 950: 'wake',\n",
       " 951: 'typically',\n",
       " 952: 'join',\n",
       " 953: 'nt',\n",
       " 954: 'external',\n",
       " 955: 'smoke',\n",
       " 956: 'direction',\n",
       " 957: 'loud',\n",
       " 958: 'expectation',\n",
       " 959: 'humor',\n",
       " 960: 'habit',\n",
       " 961: 'near',\n",
       " 962: 'pressure',\n",
       " 963: 'wife',\n",
       " 964: 'extreme',\n",
       " 965: 'organize',\n",
       " 966: 'enneagram',\n",
       " 967: 'react',\n",
       " 968: 'immediately',\n",
       " 969: 'red',\n",
       " 970: 'unheahy',\n",
       " 971: 'american',\n",
       " 972: 'apart',\n",
       " 973: 'series',\n",
       " 974: 'weak',\n",
       " 975: 'paper',\n",
       " 976: 'offend',\n",
       " 977: 'explanation',\n",
       " 978: 'evil',\n",
       " 979: 'soul',\n",
       " 980: 'attractive',\n",
       " 981: 'discover',\n",
       " 982: 'blue',\n",
       " 983: 'anger',\n",
       " 984: 'refer',\n",
       " 985: 'fan',\n",
       " 986: 'earth',\n",
       " 987: 'ready',\n",
       " 988: 'shitty',\n",
       " 989: 'safe',\n",
       " 990: 'morning',\n",
       " 991: 'abstract',\n",
       " 992: 'husband',\n",
       " 993: 'fire',\n",
       " 994: 'slightly',\n",
       " 995: 'directly',\n",
       " 996: 'neither',\n",
       " 997: 'master',\n",
       " 998: 'ah',\n",
       " 999: 'imo',\n",
       " 1000: 'scar',\n",
       " ...}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer_top_words = Tokenizer(oov_token=\"<OOV>\", split=' ', num_words=3000)\n",
    "tokenizer_top_words.fit_on_texts(X.iloc[:, 0])\n",
    "tokenizer_top_words.index_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# only 3000 words encoding\n",
    "# tmp = X.head(10)\n",
    "X_tp_words = X.copy()\n",
    "X_tp_words['tok_tw'] = X_tp_words.apply(lambda v : tokenizer_top_words.texts_to_sequences([v['posts']]), axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_tp_words['tok_tw'] = X_tp_words.apply(lambda v: np.array(v['tok_tw']).reshape(-1, 1).tolist(), axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_tp_words['tok_tw_bool']=X_tp_words.apply(lambda v: list(map(lambda t: int(t[0] > 1), v['tok_tw'])), axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Words used more than 1000 times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_dict = tokenizer_top_words.word_counts\n",
    "word_dict = OrderedDict(sorted(word_dict.items(), key = lambda t : t[-1],reverse= True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "size is 1494\n"
     ]
    }
   ],
   "source": [
    "word_dict_top = []\n",
    "for i, (key, value) in enumerate(word_dict.items()) :\n",
    "    if value >= 1000 :\n",
    "        word_dict_top.append(key)\n",
    "print(f\"size is {len(word_dict_top)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Boolean encoding\n",
    "X_freq_words = X.copy()\n",
    "X_freq_words['tok_tw'] = X_freq_words.apply(lambda v : WordPunctTokenizer().tokenize(v['posts']), axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>posts</th>\n",
       "      <th>tok_tw</th>\n",
       "      <th>tok_tw_bool</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>lot horrible flirt cue always admire enfp frie...</td>\n",
       "      <td>[lot, horrible, flirt, cue, always, admire, en...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>seem pride think great hallmark enfj make feel...</td>\n",
       "      <td>[seem, pride, think, great, hallmark, enfj, ma...</td>\n",
       "      <td>[1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>value insensitive close mind feel depress assu...</td>\n",
       "      <td>[value, insensitive, close, mind, feel, depres...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>run part gamebar gamecafé haha love yay enfjs ...</td>\n",
       "      <td>[run, part, gamebar, gamecafé, haha, love, yay...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>feel say like expose say anything understand f...</td>\n",
       "      <td>[feel, say, like, expose, say, anything, under...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21208</th>\n",
       "      <td>go end pretty much use text specific short com...</td>\n",
       "      <td>[go, end, pretty, much, use, text, specific, s...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21209</th>\n",
       "      <td>sure address concept function incidentally exa...</td>\n",
       "      <td>[sure, address, concept, function, incidentall...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21210</th>\n",
       "      <td>take part one people go fuck think horrible co...</td>\n",
       "      <td>[take, part, one, people, go, fuck, think, hor...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21211</th>\n",
       "      <td>social lesson figure ignore think people alrea...</td>\n",
       "      <td>[social, lesson, figure, ignore, think, people...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21212</th>\n",
       "      <td>weird something fond benefit sure put se first...</td>\n",
       "      <td>[weird, something, fond, benefit, sure, put, s...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21213 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   posts  \\\n",
       "0      lot horrible flirt cue always admire enfp frie...   \n",
       "1      seem pride think great hallmark enfj make feel...   \n",
       "2      value insensitive close mind feel depress assu...   \n",
       "3      run part gamebar gamecafé haha love yay enfjs ...   \n",
       "4      feel say like expose say anything understand f...   \n",
       "...                                                  ...   \n",
       "21208  go end pretty much use text specific short com...   \n",
       "21209  sure address concept function incidentally exa...   \n",
       "21210  take part one people go fuck think horrible co...   \n",
       "21211  social lesson figure ignore think people alrea...   \n",
       "21212  weird something fond benefit sure put se first...   \n",
       "\n",
       "                                                  tok_tw  \\\n",
       "0      [lot, horrible, flirt, cue, always, admire, en...   \n",
       "1      [seem, pride, think, great, hallmark, enfj, ma...   \n",
       "2      [value, insensitive, close, mind, feel, depres...   \n",
       "3      [run, part, gamebar, gamecafé, haha, love, yay...   \n",
       "4      [feel, say, like, expose, say, anything, under...   \n",
       "...                                                  ...   \n",
       "21208  [go, end, pretty, much, use, text, specific, s...   \n",
       "21209  [sure, address, concept, function, incidentall...   \n",
       "21210  [take, part, one, people, go, fuck, think, hor...   \n",
       "21211  [social, lesson, figure, ignore, think, people...   \n",
       "21212  [weird, something, fond, benefit, sure, put, s...   \n",
       "\n",
       "                                             tok_tw_bool  \n",
       "0      [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, ...  \n",
       "1      [1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, ...  \n",
       "2      [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "3      [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "4      [1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, ...  \n",
       "...                                                  ...  \n",
       "21208  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "21209  [1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, ...  \n",
       "21210  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "21211  [1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "21212  [1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "\n",
       "[21213 rows x 3 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_freq_words['tok_tw_bool'] = X_freq_words.apply(lambda row : [1 if x in row['tok_tw'] else 0 for x in word_dict_top], axis = 1)\n",
    "X_freq_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>posts</th>\n",
       "      <th>tok_tw</th>\n",
       "      <th>tok_tw_bool</th>\n",
       "      <th>tok_tw_in</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>lot horrible flirt cue always admire enfp frie...</td>\n",
       "      <td>[lot, horrible, flirt, cue, always, admire, en...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, ...</td>\n",
       "      <td>[1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>seem pride think great hallmark enfj make feel...</td>\n",
       "      <td>[seem, pride, think, great, hallmark, enfj, ma...</td>\n",
       "      <td>[1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, ...</td>\n",
       "      <td>[1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>value insensitive close mind feel depress assu...</td>\n",
       "      <td>[value, insensitive, close, mind, feel, depres...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>run part gamebar gamecafé haha love yay enfjs ...</td>\n",
       "      <td>[run, part, gamebar, gamecafé, haha, love, yay...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>feel say like expose say anything understand f...</td>\n",
       "      <td>[feel, say, like, expose, say, anything, under...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, ...</td>\n",
       "      <td>[1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21208</th>\n",
       "      <td>go end pretty much use text specific short com...</td>\n",
       "      <td>[go, end, pretty, much, use, text, specific, s...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21209</th>\n",
       "      <td>sure address concept function incidentally exa...</td>\n",
       "      <td>[sure, address, concept, function, incidentall...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, ...</td>\n",
       "      <td>[1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21210</th>\n",
       "      <td>take part one people go fuck think horrible co...</td>\n",
       "      <td>[take, part, one, people, go, fuck, think, hor...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21211</th>\n",
       "      <td>social lesson figure ignore think people alrea...</td>\n",
       "      <td>[social, lesson, figure, ignore, think, people...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21212</th>\n",
       "      <td>weird something fond benefit sure put se first...</td>\n",
       "      <td>[weird, something, fond, benefit, sure, put, s...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21213 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   posts  \\\n",
       "0      lot horrible flirt cue always admire enfp frie...   \n",
       "1      seem pride think great hallmark enfj make feel...   \n",
       "2      value insensitive close mind feel depress assu...   \n",
       "3      run part gamebar gamecafé haha love yay enfjs ...   \n",
       "4      feel say like expose say anything understand f...   \n",
       "...                                                  ...   \n",
       "21208  go end pretty much use text specific short com...   \n",
       "21209  sure address concept function incidentally exa...   \n",
       "21210  take part one people go fuck think horrible co...   \n",
       "21211  social lesson figure ignore think people alrea...   \n",
       "21212  weird something fond benefit sure put se first...   \n",
       "\n",
       "                                                  tok_tw  \\\n",
       "0      [lot, horrible, flirt, cue, always, admire, en...   \n",
       "1      [seem, pride, think, great, hallmark, enfj, ma...   \n",
       "2      [value, insensitive, close, mind, feel, depres...   \n",
       "3      [run, part, gamebar, gamecafé, haha, love, yay...   \n",
       "4      [feel, say, like, expose, say, anything, under...   \n",
       "...                                                  ...   \n",
       "21208  [go, end, pretty, much, use, text, specific, s...   \n",
       "21209  [sure, address, concept, function, incidentall...   \n",
       "21210  [take, part, one, people, go, fuck, think, hor...   \n",
       "21211  [social, lesson, figure, ignore, think, people...   \n",
       "21212  [weird, something, fond, benefit, sure, put, s...   \n",
       "\n",
       "                                             tok_tw_bool  \\\n",
       "0      [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, ...   \n",
       "1      [1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, ...   \n",
       "2      [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "3      [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "4      [1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, ...   \n",
       "...                                                  ...   \n",
       "21208  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "21209  [1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, ...   \n",
       "21210  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "21211  [1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "21212  [1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "\n",
       "                                               tok_tw_in  \n",
       "0      [1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "1      [1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, ...  \n",
       "2      [1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, ...  \n",
       "3      [1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, ...  \n",
       "4      [1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "...                                                  ...  \n",
       "21208  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, ...  \n",
       "21209  [1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, ...  \n",
       "21210  [1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, ...  \n",
       "21211  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "21212  [1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, ...  \n",
       "\n",
       "[21213 rows x 4 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_freq_words['tok_tw_in'] = X_freq_words.apply(lambda row : [1 if x in word_dict_top else 0 for x in row['tok_tw']], axis = 1)\n",
    "X_freq_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataframe print"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>posts</th>\n",
       "      <th>tok_tw</th>\n",
       "      <th>tok_tw_bool</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>lot horrible flirt cue always admire enfp frie...</td>\n",
       "      <td>[[32], [1145], [1410], [2420], [45], [1687], [...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>seem pride think great hallmark enfj make feel...</td>\n",
       "      <td>[[49], [1814], [3], [97], [1], [532], [7], [9]...</td>\n",
       "      <td>[1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>value insensitive close mind feel depress assu...</td>\n",
       "      <td>[[186], [1], [160], [90], [9], [690], [354], [...</td>\n",
       "      <td>[1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>run part gamebar gamecafé haha love yay enfjs ...</td>\n",
       "      <td>[[267], [108], [1], [1], [263], [34], [2149], ...</td>\n",
       "      <td>[1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>feel say like expose say anything understand f...</td>\n",
       "      <td>[[9], [13], [2], [2067], [13], [71], [51], [9]...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21208</th>\n",
       "      <td>go end pretty much use text specific short com...</td>\n",
       "      <td>[[11], [94], [59], [22], [33], [701], [455], [...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21209</th>\n",
       "      <td>sure address concept function incidentally exa...</td>\n",
       "      <td>[[62], [1203], [519], [60], [1], [187], [15], ...</td>\n",
       "      <td>[1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21210</th>\n",
       "      <td>take part one people go fuck think horrible co...</td>\n",
       "      <td>[[25], [108], [10], [4], [11], [133], [3], [11...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21211</th>\n",
       "      <td>social lesson figure ignore think people alrea...</td>\n",
       "      <td>[[126], [1478], [243], [561], [3], [4], [219],...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21212</th>\n",
       "      <td>weird something fond benefit sure put se first...</td>\n",
       "      <td>[[304], [23], [1], [731], [62], [114], [245], ...</td>\n",
       "      <td>[1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21213 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   posts  \\\n",
       "0      lot horrible flirt cue always admire enfp frie...   \n",
       "1      seem pride think great hallmark enfj make feel...   \n",
       "2      value insensitive close mind feel depress assu...   \n",
       "3      run part gamebar gamecafé haha love yay enfjs ...   \n",
       "4      feel say like expose say anything understand f...   \n",
       "...                                                  ...   \n",
       "21208  go end pretty much use text specific short com...   \n",
       "21209  sure address concept function incidentally exa...   \n",
       "21210  take part one people go fuck think horrible co...   \n",
       "21211  social lesson figure ignore think people alrea...   \n",
       "21212  weird something fond benefit sure put se first...   \n",
       "\n",
       "                                                  tok_tw  \\\n",
       "0      [[32], [1145], [1410], [2420], [45], [1687], [...   \n",
       "1      [[49], [1814], [3], [97], [1], [532], [7], [9]...   \n",
       "2      [[186], [1], [160], [90], [9], [690], [354], [...   \n",
       "3      [[267], [108], [1], [1], [263], [34], [2149], ...   \n",
       "4      [[9], [13], [2], [2067], [13], [71], [51], [9]...   \n",
       "...                                                  ...   \n",
       "21208  [[11], [94], [59], [22], [33], [701], [455], [...   \n",
       "21209  [[62], [1203], [519], [60], [1], [187], [15], ...   \n",
       "21210  [[25], [108], [10], [4], [11], [133], [3], [11...   \n",
       "21211  [[126], [1478], [243], [561], [3], [4], [219],...   \n",
       "21212  [[304], [23], [1], [731], [62], [114], [245], ...   \n",
       "\n",
       "                                             tok_tw_bool  \n",
       "0      [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "1      [1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "2      [1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, ...  \n",
       "3      [1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, ...  \n",
       "4      [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "...                                                  ...  \n",
       "21208  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, ...  \n",
       "21209  [1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "21210  [1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, ...  \n",
       "21211  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "21212  [1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, ...  \n",
       "\n",
       "[21213 rows x 3 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_tp_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>posts</th>\n",
       "      <th>tok_tw</th>\n",
       "      <th>tok_tw_bool</th>\n",
       "      <th>tok_tw_in</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>lot horrible flirt cue always admire enfp frie...</td>\n",
       "      <td>[lot, horrible, flirt, cue, always, admire, en...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, ...</td>\n",
       "      <td>[1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>seem pride think great hallmark enfj make feel...</td>\n",
       "      <td>[seem, pride, think, great, hallmark, enfj, ma...</td>\n",
       "      <td>[1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, ...</td>\n",
       "      <td>[1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>value insensitive close mind feel depress assu...</td>\n",
       "      <td>[value, insensitive, close, mind, feel, depres...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>run part gamebar gamecafé haha love yay enfjs ...</td>\n",
       "      <td>[run, part, gamebar, gamecafé, haha, love, yay...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>feel say like expose say anything understand f...</td>\n",
       "      <td>[feel, say, like, expose, say, anything, under...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, ...</td>\n",
       "      <td>[1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21208</th>\n",
       "      <td>go end pretty much use text specific short com...</td>\n",
       "      <td>[go, end, pretty, much, use, text, specific, s...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21209</th>\n",
       "      <td>sure address concept function incidentally exa...</td>\n",
       "      <td>[sure, address, concept, function, incidentall...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, ...</td>\n",
       "      <td>[1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21210</th>\n",
       "      <td>take part one people go fuck think horrible co...</td>\n",
       "      <td>[take, part, one, people, go, fuck, think, hor...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21211</th>\n",
       "      <td>social lesson figure ignore think people alrea...</td>\n",
       "      <td>[social, lesson, figure, ignore, think, people...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21212</th>\n",
       "      <td>weird something fond benefit sure put se first...</td>\n",
       "      <td>[weird, something, fond, benefit, sure, put, s...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21213 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   posts  \\\n",
       "0      lot horrible flirt cue always admire enfp frie...   \n",
       "1      seem pride think great hallmark enfj make feel...   \n",
       "2      value insensitive close mind feel depress assu...   \n",
       "3      run part gamebar gamecafé haha love yay enfjs ...   \n",
       "4      feel say like expose say anything understand f...   \n",
       "...                                                  ...   \n",
       "21208  go end pretty much use text specific short com...   \n",
       "21209  sure address concept function incidentally exa...   \n",
       "21210  take part one people go fuck think horrible co...   \n",
       "21211  social lesson figure ignore think people alrea...   \n",
       "21212  weird something fond benefit sure put se first...   \n",
       "\n",
       "                                                  tok_tw  \\\n",
       "0      [lot, horrible, flirt, cue, always, admire, en...   \n",
       "1      [seem, pride, think, great, hallmark, enfj, ma...   \n",
       "2      [value, insensitive, close, mind, feel, depres...   \n",
       "3      [run, part, gamebar, gamecafé, haha, love, yay...   \n",
       "4      [feel, say, like, expose, say, anything, under...   \n",
       "...                                                  ...   \n",
       "21208  [go, end, pretty, much, use, text, specific, s...   \n",
       "21209  [sure, address, concept, function, incidentall...   \n",
       "21210  [take, part, one, people, go, fuck, think, hor...   \n",
       "21211  [social, lesson, figure, ignore, think, people...   \n",
       "21212  [weird, something, fond, benefit, sure, put, s...   \n",
       "\n",
       "                                             tok_tw_bool  \\\n",
       "0      [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, ...   \n",
       "1      [1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, ...   \n",
       "2      [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "3      [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "4      [1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, ...   \n",
       "...                                                  ...   \n",
       "21208  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "21209  [1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, ...   \n",
       "21210  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "21211  [1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "21212  [1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "\n",
       "                                               tok_tw_in  \n",
       "0      [1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "1      [1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, ...  \n",
       "2      [1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, ...  \n",
       "3      [1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, ...  \n",
       "4      [1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "...                                                  ...  \n",
       "21208  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, ...  \n",
       "21209  [1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, ...  \n",
       "21210  [1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, ...  \n",
       "21211  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "21212  [1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, ...  \n",
       "\n",
       "[21213 rows x 4 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_freq_words"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.7 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "ad2bdc8ecc057115af97d19610ffacc2b4e99fae6737bb82f5d7fb13d2f2c186"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}