02-02_conference_calls_preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Confrence Calls - Clean Sentences\r\n",
    "*Convert parsed transcripts to cleaned sentences*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "NAME = '02-02_conference_calls_preprocess'\n",
    "PROJECT = 'conference-calls-sentiment'\n",
    "PYTHON_VERSION = '3.7.0'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# NLP\n",
    "from nltk.tokenize import sent_tokenize\n",
    "\n",
    "# Logging\n",
    "from utils import log_step "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "workdir = re.sub(\"(?<={})[\\w\\W]*\".format(PROJECT), \"\", os.getcwd())\n",
    "os.chdir(workdir)\n",
    "\n",
    "pipeline = os.path.join('2_pipeline', NAME)\n",
    "if not os.path.exists(pipeline):\n",
    "    os.makedirs(pipeline)\n",
    "    for folder in ['out', 'store', 'tmp']:\n",
    "        os.makedirs(os.path.join(pipeline, folder))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Main Code "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PARSED_TRANSCRIPTS_PATH = os.path.join('2_pipeline', '02-01_conference_calls_parse_txt.', 'out', 'cc_transcripts_parsed.feather')\n",
    "parsed_transcripts = pd.read_feather(PARSED_TRANSCRIPTS_PATH)\n",
    "parsed_transcripts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "@log_step\n",
    "def start_pipeline(transcripts):\n",
    "    return transcripts.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "@log_step\n",
    "def filter_qa(transcripts):\n",
    "    transcripts_qa = transcripts[transcripts['section_name'] == 'Questions and Answers']\n",
    "    return transcripts_qa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "@log_step\n",
    "def select_speaker_roles(transcripts):\n",
    "    mapping = {r'\\bC[A-Z]O\\b|chief|officer': 'Management',\n",
    "               r'analyst': 'Analyst'}\n",
    "\n",
    "    for role, clean_role in mapping.items():\n",
    "        transcripts.loc[transcripts['speaker_role'].str.contains(role, case=False), 'speaker_role_clean'] = clean_role\n",
    "    \n",
    "    transcripts = transcripts.dropna(subset=['speaker_role_clean'])\n",
    "    return transcripts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "@log_step\n",
    "def tokenize_sentences(transcripts):\n",
    "    return (transcripts.loc[transcripts['transcript'].notna()]\n",
    "            .reset_index(drop=True)\n",
    "            .assign(transcript=lambda x: x['transcript'].apply(sent_tokenize))\n",
    "            .explode('transcript')\n",
    "            .dropna(subset=['transcript'])\n",
    "            .reset_index(drop=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "TAGS_PATTERN = re.compile(r'[<\\(\\[].*?[>\\)\\]]')\r\n",
    "WORDS_WITH_NUMBERS = re.compile(r'\\w*\\d\\w*')\r\n",
    "REPLACE_BY_SPACE = re.compile(r'[/(){}\\[\\]\\|@,;]')\r\n",
    "BAD_SYMBOLS = re.compile(r'[^a-z ]')\r\n",
    "\r\n",
    "def process_transcript(transcript):\r\n",
    "    transcript_clean = transcript.lower()\r\n",
    "    transcript_clean = TAGS_PATTERN.sub('', transcript_clean)\r\n",
    "    transcript_clean = WORDS_WITH_NUMBERS.sub('', transcript_clean)\r\n",
    "    transcript_clean = REPLACE_BY_SPACE.sub(' ', transcript_clean)\r\n",
    "    transcript_clean = BAD_SYMBOLS.sub('', transcript_clean)\r\n",
    "    transcript_clean = re.sub('\\s+', ' ', transcript_clean).strip()  # Remove extra whitespace characters\r\n",
    "    return transcript_clean\r\n",
    "\r\n",
    "def convert_name(speaker_name):\r\n",
    "    '''Converts name to I/B/E/S format'''\r\n",
    "    speaker_name = speaker_name.upper()\r\n",
    "    surname = speaker_name.split()[-1]\r\n",
    "    first_name = speaker_name.split()[0]\r\n",
    "    return ' '.join([surname, first_name[0]])\r\n",
    "\r\n",
    "@log_step\r\n",
    "def process_text(transcripts):\r\n",
    "    return transcripts.assign(transcript=lambda x: x['transcript'].apply(process_transcript),\r\n",
    "                              speaker_name=lambda x: x['speaker_name'].apply(convert_name),\r\n",
    "                              speaker_firm=lambda x: x['speaker_firm'].str.upper())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "@log_step\n",
    "def remove_short_sentences(transcripts):\n",
    "    return (transcripts\n",
    "            .assign(num_words=lambda x: x['transcript'].str.split().str.len())\n",
    "            .query('num_words >= 5'))  # Sentences should have at least 5 words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "@log_step\n",
    "def clean_data(transcripts):\n",
    "    return (transcripts\n",
    "            .assign(quarter=lambda x: x['event_date'].dt.to_period('Q'))\n",
    "            .filter([\n",
    "                'gvkey', 'ticker', 'event_date', 'coname', 'speaker_role_clean',\n",
    "                'speaker_name', 'speaker_firm', 'transcript', 'num_words',\n",
    "                'transcript_id', 'quarter', 'year'\n",
    "                ])\n",
    "            .rename(columns={'speaker_role_clean': 'speaker_role'})\n",
    "            .sort_values(['gvkey', 'event_date', 'speaker_name'])\n",
    "            .reset_index(drop=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_transcripts = (parsed_transcripts\n",
    "                       .pipe(start_pipeline)\n",
    "                       .pipe(filter_qa)\n",
    "                       .pipe(select_speaker_roles)\n",
    "                       .pipe(tokenize_sentences)\n",
    "                       .pipe(process_text)\n",
    "                       .pipe(remove_short_sentences)\n",
    "                       .pipe(clean_data))\n",
    "                       \n",
    "cleaned_transcripts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_transcripts.to_feather(os.path.join(pipeline, 'out', 'cc_transcripts.feather'))"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "4ad8bff83ca7a203c8f5c1f2f3a00576dbcaaca98e752b3367803050219b2dc1"
  },
  "kernelspec": {
   "display_name": "Python 3.7.0 64-bit ('ccs': conda)",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": ""
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}