notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "import pandas as pd\n",
    "import plotly.express as px\n",
    "import plotly.offline as py\n",
    "\n",
    "from itertools import combinations\n",
    "from os import mkdir\n",
    "from os.path import exists\n",
    "from re import findall\n",
    "from string import punctuation\n",
    "from time import time\n",
    "\n",
    "py.init_notebook_mode(connected=True)\n",
    "\n",
    "t0 = time()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cleanup functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_punct(word):\n",
    "    ''' Remove punctuation from word. '''\n",
    "    for punct in punctuation:\n",
    "        word = word.strip(punct)\n",
    "    return word\n",
    "\n",
    "def is_number(word):\n",
    "    ''' Return True if word is a number. '''\n",
    "    try:\n",
    "        float(word)\n",
    "    except:\n",
    "        return False\n",
    "    return True\n",
    "\n",
    "def find_mentions(sentence):\n",
    "    ''' Return list of mentions from string. '''\n",
    "    mentions = {}\n",
    "    chunks = sentence.split()\n",
    "    \n",
    "    for i, word in enumerate(chunks):\n",
    "        if len(word) > 1\\\n",
    "        and not is_number(word)\\\n",
    "        and word in (word.capitalize(), word.upper()):\n",
    "            if (i-1) in mentions:\n",
    "                mentions[i] = ' '.join([mentions.pop(i-1), word])\n",
    "            else:\n",
    "                mentions[i] = word\n",
    "    \n",
    "    return list(mentions.values())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data\n",
    "\n",
    "Build a `Pandas.DataFrame` object from desired `columns` only (smaller memory footprint)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name = \"\" # <-- data set in CSV format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(file_name)\n",
    "\n",
    "print(f'Loaded {df.shape[0]} rows and {df.shape[1]} columns.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Page stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not exists('RESULTS'):\n",
    "    mkdir('RESULTS')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = df[['Likes', 'Comments', 'Shares', 'Love', 'Wow', 'Haha', 'Sad', 'Angry', 'Care']]\n",
    "\n",
    "with pd.option_context('mode.chained_assignment', None):\n",
    "    stats['Total'] = stats.sum(axis=1)\n",
    "\n",
    "try: stats.index = df['Page Name']\n",
    "except: stats.index = df['Group Name']\n",
    "\n",
    "stats.to_csv('RESULTS/stats.csv')\n",
    "\n",
    "stats.sort_values('Total', ascending=False).head(25) # <-- top 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build graphs\n",
    "\n",
    "Prepare edge list `E` and build directed graphs `G{1,2,3}` using `NetworkX.DiGraph`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Graph from links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "try: E1 = df[['Page Name', 'Link']]\n",
    "except: E1 = df[['Group Name', 'Link']]\n",
    "\n",
    "E1.to_csv('RESULTS/E1_Links.csv')\n",
    "E1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "G1 = nx.DiGraph()\n",
    "G1.add_edges_from(E1.values)\n",
    "\n",
    "nx.set_node_attributes(G1, stats)\n",
    "nx.write_gexf(G1, 'RESULTS/G1_Links.gexf')\n",
    "\n",
    "print(f'Graph from links has {G1.order()} nodes and {G1.size()} edges.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Graph from mentions in link text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "try: E2 = df[['Page Name', 'Link Text']]\n",
    "except: E2 = df[['Group Name', 'Link Text']]\n",
    "\n",
    "with pd.option_context('mode.chained_assignment', None):\n",
    "    E2['Mentions'] = E2['Link Text'].apply(lambda x: find_mentions(x) if isinstance(x, str) else None)\n",
    "    E2['Mentions'] = [None if x and x[0].split()[0] == 'Timeline' else x for x in E2['Mentions']]\n",
    "    E2['Mentions'] = [None if x == ['This'] else x for x in E2['Mentions']]\n",
    "    E2.dropna(inplace=True, subset=['Mentions'])\n",
    "    E2.to_csv('RESULTS/E2_Mentions.csv')\n",
    "\n",
    "E2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "G2 = nx.DiGraph()\n",
    "try: G2.add_edges_from(E2[['Page Name', 'Mentions']].explode('Mentions').dropna().values)\n",
    "except: G2.add_edges_from(E2[['Group Name', 'Mentions']].explode('Mentions').dropna().values)\n",
    "\n",
    "nx.set_node_attributes(G2, stats)\n",
    "nx.write_gexf(G2, 'RESULTS/G2_Mentions.gexf')\n",
    "\n",
    "print(f'Graph from mentions has {G2.order()} nodes and {G2.size()} edges.')\n",
    "pd.DataFrame(nx.degree(G2)).sort_values(1, ascending=False).head(25) # <-- top 25"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Graph from hashtags in message and description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try: E3 = df[['Page Name', 'Message', 'Description']]\n",
    "except: E3 = df[['Group Name', 'Message', 'Description']]\n",
    "\n",
    "with pd.option_context('mode.chained_assignment', None):\n",
    "    E3['text'] = [str(x) for x in E3.values]\n",
    "    E3['hashtags'] = [findall(r'\\s([#][\\w_-]+)', str(x)) for x in E3['text']]\n",
    "    E3['hashtags'] = [None if x == [] else x for x in E3['hashtags'].values]\n",
    "    E3.dropna(inplace=True, subset=['hashtags'])\n",
    "    E3.to_csv('RESULTS/E3_Hashtags.csv')\n",
    "\n",
    "E3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "G3 = nx.DiGraph()\n",
    "try: G3.add_edges_from(E3[['Page Name', 'hashtags']].explode('hashtags').values)\n",
    "except: G3.add_edges_from(E3[['Group Name', 'hashtags']].explode('hashtags').values)\n",
    "\n",
    "nx.set_node_attributes(G3, stats)\n",
    "nx.write_gexf(G3, 'RESULTS/G3_Hashtags.gexf')\n",
    "\n",
    "print(f'Graph from hashtags has {G3.order()} nodes and {G3.size()} edges.')\n",
    "pd.DataFrame(nx.degree(G3)).sort_values(1, ascending=False).head(25) # <-- top 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "G4 = nx.DiGraph()\n",
    "\n",
    "for hashtags in E3['hashtags'].values.tolist():\n",
    "    G4.add_edges_from(combinations(hashtags, 2))\n",
    "\n",
    "nx.set_node_attributes(G4, stats)\n",
    "nx.write_gexf(G4, 'RESULTS/G4_Hashtags-Hashtags.gexf')\n",
    "\n",
    "print(f'Graph from concurrent hashtags has {G4.order()} nodes and {G4.size()} edges.')\n",
    "pd.DataFrame(nx.degree(G4)).sort_values(1, ascending=False).head(25) # <-- top 25"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'done in {time()-t0:.3f}s')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}