RedditDataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "reddit_dir = Path.cwd() / 'datasets/reddit/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# to collect sufficient training data, since one is not enough"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.read_json(reddit_dir / 'RC_2016-01', lines=True)\n",
    "tuned_df1 = df1[['author', 'body', 'created_utc', 'score',  'subreddit']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.read_json(reddit_dir / 'RC_2016-02', lines=True)\n",
    "tuned_df2 = df2[['author', 'body', 'created_utc', 'score',  'subreddit']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3 = pd.read_json(reddit_dir / 'RC_2016-03', lines=True)\n",
    "tuned_df3 = df3[['author', 'body', 'created_utc', 'score',  'subreddit']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tuned_df1.dropna(axis=0, inplace=True) # drop rows where NAN exists\n",
    "tuned_df1['body'].replace(to_replace=[r\"\\\\t|\\\\n|\\\\r\", \"\\t|\\n|\\r\"], value=[\"\",\"\"], regex=True, inplace=True)\n",
    "tuned_df1.rename(columns={'author': 'UserName', 'body': 'Text', 'created_utc': 'Timestamp', 'score': 'Score', 'ups': 'UpScore', 'subreddit': 'Categories'}, \n",
    "                        inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tuned_df2.dropna(axis=0, inplace=True) # drop rows where NAN exists\n",
    "tuned_df2['body'].replace(to_replace=[r\"\\\\t|\\\\n|\\\\r\", \"\\t|\\n|\\r\"], value=[\"\",\"\"], regex=True, inplace=True)\n",
    "tuned_df2.rename(columns={'author': 'UserName', 'body': 'Text', 'created_utc': 'Timestamp', 'score': 'Score', 'ups': 'UpScore', 'subreddit': 'Categories'}, \n",
    "                        inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tuned_df3.dropna(axis=0, inplace=True) # drop rows where NAN exists\n",
    "tuned_df3['body'].replace(to_replace=[r\"\\\\t|\\\\n|\\\\r\", \"\\t|\\n|\\r\"], value=[\"\",\"\"], regex=True, inplace=True)\n",
    "tuned_df3.rename(columns={'author': 'UserName', 'body': 'Text', 'created_utc': 'Timestamp', 'score': 'Score', 'ups': 'UpScore', 'subreddit': 'Categories'}, \n",
    "                        inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# if full dataset exists, reading it directly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tuned_df = pd.read_csv(reddit_dir / 'html_parsed_full_dataset.csv', sep='|')\n",
    "tuned_df.rename(columns={'author': 'UserName', 'body': 'Text', 'created_utc': 'Timestamp', 'score': 'Score', 'ups': 'UpScore', 'subreddit': 'Categories'}, \n",
    "                        inplace=True)\n",
    "tuned_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tuned_df = tuned_df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp_df = tuned_df['UserName']\n",
    "tmp_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "disclosed_df = pd.read_csv(reddit_dir / 'disclosed_dataset.csv', usecols=['Categories'])\n",
    "undisclosed_df = pd.read_csv(reddit_dir / 'undisclosed_dataset.csv', usecols=['Categories'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_df = pd.concat([disclosed_df, undisclosed_df], axis=0, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_df = concat_df.dropna()\n",
    "categories_column = concat_df.Categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "category_freq = {}\n",
    "for cate in categories_column:\n",
    "    for sub_cate in cate.lower().split(','):\n",
    "        sub_cate = sub_cate.strip()\n",
    "        category_freq[sub_cate] = category_freq.get(sub_cate, 0) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_category_freq = sorted(category_freq.items(), key=lambda kv: kv[1], reverse=True)\n",
    "with open(reddit_dir / 'top_10_categories.json', 'w') as f:\n",
    "    json.dump(sorted_category_freq[:10], f)\n",
    "len(sorted_category_freq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_category_freq"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# html parser for body text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "warnings.filterwarnings(\"ignore\", category=UserWarning, module='bs4')\n",
    "tuned_df1['Text'] = tuned_df1['Text'].apply(lambda x: BeautifulSoup(x, 'html.parser').get_text().lower())\n",
    "tuned_df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "warnings.filterwarnings(\"ignore\", category=UserWarning, module='bs4')\n",
    "tuned_df2['Text'] = tuned_df2['Text'].apply(lambda x: BeautifulSoup(x, 'html.parser').get_text().lower())\n",
    "tuned_df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "warnings.filterwarnings(\"ignore\", category=UserWarning, module='bs4')\n",
    "tuned_df3['Text'] = tuned_df3['Text'].apply(lambda x: BeautifulSoup(x, 'html.parser').get_text().lower())\n",
    "tuned_df3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tuned_df.to_csv(reddit_dir / 'html_parsed_full_dataset.csv', index=False, sep='|')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tuned_df = pd.read_csv(reddit_dir / 'html_parsed_full_dataset.csv', sep='|')\n",
    "# tuned_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gender_guesser.detector as gender\n",
    "\n",
    "d = gender.Detector(case_sensitive=False)\n",
    "\n",
    "import string\n",
    "translator = str.maketrans('', '', string.punctuation)\n",
    "\n",
    "def get_gender(name):\n",
    "    try:\n",
    "        n = name.lower()\n",
    "        if ('mom' in n) or ('girl' in n) or ('angel' in n) or ('mum' in n) or ('mother' in n) or ('woman' in n):\n",
    "            return 'female'\n",
    "        if ('boy' in n) or ('dude' in n):\n",
    "            return 'male'\n",
    "        temp = name.translate(translator).split()\n",
    "    except:\n",
    "        return 'unknown'\n",
    "    \n",
    "    if len(temp) > 0:\n",
    "        first_name = temp[0]\n",
    "        first_name = ''.join([i for i in first_name if not i.isdigit()])\n",
    "        gender = d.get_gender(first_name,'usa')\n",
    "        return gender\n",
    "    else:\n",
    "        return 'unknown'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tuned_df1['Gender'] = tuned_df1['UserName'].apply(lambda x:get_gender(x))\n",
    "tuned_df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "del tuned_df1\n",
    "gc.collect()\n",
    "tuned_df2['Gender'] = tuned_df2['UserName'].apply(lambda x:get_gender(x))\n",
    "tuned_df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "del tuned_df2\n",
    "gc.collect()\n",
    "tuned_df3['Gender'] = tuned_df3['UserName'].apply(lambda x:get_gender(x))\n",
    "tuned_df3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "disclosed_flags1 = tuned_df1['Gender'].isin(['female', 'male'])\n",
    "undisclosed_flags1 = ~disclosed_flags1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "disclosed_flags2 = tuned_df2['Gender'].isin(['female', 'male'])\n",
    "undisclosed_flags2 = ~disclosed_flags2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "disclosed_flags3 = tuned_df3['Gender'].isin(['female', 'male'])\n",
    "undisclosed_flags3 = ~disclosed_flags3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "disclosed_gender_df1 = tuned_df1[disclosed_flags1]\n",
    "disclosed_gender_df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "disclosed_gender_df1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "disclosed_gender_df1.to_csv(reddit_dir / 'disclosed_dataset1.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "disclosed_gender_df2 = tuned_df2[disclosed_flags2]\n",
    "disclosed_gender_df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "disclosed_gender_df2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "disclosed_gender_df2.to_csv(reddit_dir / 'disclosed_dataset2.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "disclosed_gender_df3 = tuned_df3[disclosed_flags3]\n",
    "disclosed_gender_df3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "disclosed_gender_df3.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "disclosed_gender_df3.to_csv(reddit_dir / 'disclosed_dataset3.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "disclosed_gender_df1 = pd.read_csv(reddit_dir / 'disclosed_dataset1.csv')\n",
    "disclosed_gender_df2 = pd.read_csv(reddit_dir / 'disclosed_dataset2.csv')\n",
    "disclosed_gender_df3 = pd.read_csv(reddit_dir / 'disclosed_dataset3.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "disclosed_df = pd.concat([disclosed_gender_df1, disclosed_gender_df2, disclosed_gender_df3], axis=0, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "disclosed_df.shape # at least (2117828, 7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "disclosed_df.to_csv(reddit_dir / 'disclosed_dataset.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# undisclosed data is quite enough, so using one of them \n",
    "undisclosed_gender_df1 = tuned_df[undisclosed_flags1]\n",
    "undisclosed_gender_df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "undisclosed_gender_df3 = tuned_df3[undisclosed_flags3]\n",
    "undisclosed_gender_df3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "undisclosed_gender_df3.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "undisclosed_gender_df3.to_csv(reddit_dir / 'undisclosed_dataset.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "undisclosed_gender_df = undisclosed_gender_df3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# write or read csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os.path\n",
    "import pandas as pd\n",
    "if not os.path.exists(reddit_dir / 'disclosed_dataset.csv'):\n",
    "    print('saving disclosed dataset to csv')\n",
    "    disclosed_df.to_csv(reddit_dir / 'disclosed_dataset.csv', index=False)\n",
    "else:\n",
    "    disclosed_gender_df = pd.read_csv(reddit_dir / 'disclosed_dataset.csv')\n",
    "\n",
    "if not os.path.exists(reddit_dir / 'undisclosed_dataset.csv'):\n",
    "    print('saving undisclosed dataset to csv')\n",
    "    undisclosed_gender_df1.to_csv(reddit_dir / 'undisclosed_dataset.csv', index=False)\n",
    "else:\n",
    "    undisclosed_gender_df = pd.read_csv(reddit_dir / 'undisclosed_dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(disclosed_gender_df.UserName.unique().shape, disclosed_gender_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping = {'female' : 1, 'male' : 0}\n",
    "disclosed_dataset_df = disclosed_df[['UserName', 'Text', 'Gender']]\n",
    "disclosed_dataset_df.replace({'Gender': mapping}, inplace=True)\n",
    "disclosed_dataset_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# plot data distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping = {'female' : 1, 'male' : 0}\n",
    "disclosed_gender_df.replace({'Gender': mapping}, inplace=True)\n",
    "disclosed_gender_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# disclosed_gender_df = disclosed_dataset_df\n",
    "DF_shape = disclosed_gender_df.loc[disclosed_gender_df['Gender'] == 1].shape\n",
    "DM_shape = disclosed_gender_df.loc[disclosed_gender_df['Gender'] == 0].shape\n",
    "UNDIS_shape = undisclosed_gender_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_size = disclosed_gender_df.shape[0] + UNDIS_shape[0]\n",
    "print(total_size, disclosed_gender_df.shape[0], UNDIS_shape[0], DM_shape[0], DF_shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DM_shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "labels = 'SM', 'SW', 'Performing'\n",
    "sizes = [DM_shape[0]/total_size, DF_shape[0]/total_size, UNDIS_shape[0]/total_size]\n",
    "explode = (0, 0, 0)  # only \"explode\" the 2nd slice (i.e. 'Hogs')\n",
    "\n",
    "fig1, ax1 = plt.subplots()\n",
    "ax1.pie(sizes, explode=explode, labels=labels, autopct='%1.1f%%',\n",
    "        shadow=True, startangle=10)\n",
    "ax1.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.\n",
    "\n",
    "plt.savefig(\"reddit_data_distribution.png\", dpi=400)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split disclosed dataset into train, test and validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split disclosed dataset into train, test and validation\n",
    "# train: text, gender\n",
    "# test: name, text, gender\n",
    "# validation: text, gender"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from pathlib import Path\n",
    "reddit_dir = Path.cwd() / 'datasets/reddit/'\n",
    "\n",
    "\n",
    "if 'disclosed_dataset_df' in locals():\n",
    "    print('existed.')\n",
    "    # 80/20 train/test\n",
    "    train_df, test_df = train_test_split(disclosed_dataset_df, test_size=0.2)\n",
    "    # 80/20 train/validation\n",
    "    train_df, validation_df = train_test_split(train_df, test_size=0.2)\n",
    "else:\n",
    "    disclosed_dataset_df = pd.read_csv(reddit_dir / 'disclosed_dataset.csv')\n",
    "    # 80/20 train/test\n",
    "    train_df, test_df = train_test_split(disclosed_dataset_df, test_size=0.2)\n",
    "    # 80/20 train/validation\n",
    "    train_df, validation_df = train_test_split(train_df, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extracting related attributes for training, validation and test\n",
    "\n",
    "# gender_map = {'male': 0, 'female': 1}\n",
    "train_gender_text_df = train_df[['Gender', 'Text']]\n",
    "# train_gender_text_df.replace({'Gender': gender_map}, inplace=True)\n",
    "validation_gender_text_df = validation_df[['Gender', 'Text']]\n",
    "# validation_gender_text_df.replace({'Gender': gender_map}, inplace=True)\n",
    "test_name_text_gender_df = test_df[['UserName', 'Text', 'Gender']]\n",
    "# test_name_text_gender_df.replace({'Gender': gender_map}, inplace=True)\n",
    "\n",
    "# train_gender_text_df.replace(to_replace=[r\"\\\\t|\\\\n|\\\\r\", \"\\t|\\n|\\r\"], value=[\"\",\"\"], regex=True, inplace=True)\n",
    "train_gender_text_df.to_csv(reddit_dir / 'training_gender_text.csv', index=False, header=False)\n",
    "\n",
    "# validation_gender_text_df.replace(to_replace=[r\"\\\\t|\\\\n|\\\\r\", \"\\t|\\n|\\r\"], value=[\"\",\"\"], regex=True, inplace=True)\n",
    "validation_gender_text_df.to_csv(reddit_dir / 'validation_gender_text.csv', index=False, header=False)\n",
    "\n",
    "# test_name_text_gender_df.replace(to_replace=[r\"\\\\t|\\\\n|\\\\r\", \"\\t|\\n|\\r\"], value=[\"\",\"\"], regex=True, inplace=True)\n",
    "test_name_text_gender_df.to_csv(reddit_dir / 'test_name_text_gender.csv', index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# undisclosed_gender_df.rename(columns={'body': 'Text'}, inplace=True)\n",
    "# undisclosed_gender_df = pd.read_csv(reddit_dir / 'undisclosed_dataset.csv')\n",
    "undisclosed_gender_df = undisclosed_gender_df[['UserName', 'Text', 'Gender']]\n",
    "# undisclosed_gender_df.replace({'Gender': gender_map}, inplace=True)\n",
    "# undisclosed_gender_df.replace(to_replace=[r\"\\\\t|\\\\n|\\\\r\", \"\\t|\\n|\\r\"], value=[\"\",\"\"], regex=True, inplace=True)\n",
    "undisclosed_gender_df.to_csv(reddit_dir / 'undisclosed_id_text_gender.csv', index=False, header=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Only using when forget to map gender to numerical number 0, 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import pandas as pd\n",
    "reddit_dir = Path.cwd() / 'datasets/reddit/'\n",
    "train_gender_text_df = pd.read_csv(reddit_dir / 'training_gender_text.csv', names=['Gender', 'Text'])\n",
    "validation_gender_text_df = pd.read_csv(reddit_dir / 'validation_gender_text.csv', names=['Gender', 'Text'])\n",
    "\n",
    "gender_map = {'male': 0, 'female': 1}\n",
    "train_gender_text_df.replace({'Gender': gender_map}, inplace=True)\n",
    "validation_gender_text_df.replace({'Gender': gender_map}, inplace=True)\n",
    "\n",
    "train_gender_text_df.to_csv(reddit_dir / 'training_gender_text_mapped.csv', index=False, header=False)\n",
    "validation_gender_text_df.to_csv(reddit_dir / 'validation_gender_text_mapped.csv', index=False, header=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import pandas as pd\n",
    "\n",
    "reddit_dir = Path.cwd() / 'datasets/reddit/'\n",
    "undisclosed_id_gender_text_df = pd.read_csv(reddit_dir / 'undisclosed_id_text_gender.csv', names=['UserName', 'Text', 'Gender'])\n",
    "\n",
    "gender_map = {'male': 0, 'female': 1}\n",
    "train_gender_text_df.replace({'Gender': gender_map}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# accuracy: 0.7296723086106249 at epoch: 9"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# reprocess undisclosed dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not Path(reddit_dir / 'undisclosed_dataset.csv').exists():\n",
    "    print('saving undisclosed dataset to csv')\n",
    "    undisclosed_gender_df.to_csv(reddit_dir / 'undisclosed_dataset.csv', index=False)\n",
    "else:\n",
    "    undisclosed_gender_df = pd.read_csv(reddit_dir / 'undisclosed_dataset.csv')\n",
    "    \n",
    "undisclosed_gender_df.rename(columns={'body': 'Text'}, inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "undisclosed_gender_df = undisclosed_gender_df[['UserName', 'Text', 'Gender']]\n",
    "undisclosed_gender_df.replace(to_replace=[r\"\\\\t|\\\\n|\\\\r\", \"\\t|\\n|\\r\"], value=[\"\",\"\"], regex=True, inplace=True)\n",
    "undisclosed_gender_df.to_csv(reddit_dir / 'undisclosed_id_text_gender.csv', index=False, header=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}