dependency_parsing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f701604a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# setup for google colab\n",
    "\n",
    "# !pip install spacy==3.1.4\n",
    "# !python -m spacy download en_core_web_md\n",
    "\n",
    "# restart runtime before running notebook  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a6de9fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "from collections import Counter\n",
    "\n",
    "import spacy\n",
    "from spacy import displacy\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa4720b8",
   "metadata": {},
   "source": [
    "# Dependency parsing with SpaCy"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3c89e7a",
   "metadata": {},
   "source": [
    "### Load model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9b1a302",
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en_core_web_md\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c83b1455",
   "metadata": {},
   "source": [
    "### Load and filter data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67959a41",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/dataset.csv')\n",
    "dataset = df[df['query'] == 'happy+white+woman']\n",
    "dataset.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4788290",
   "metadata": {},
   "outputs": [],
   "source": [
    "google_dataset = dataset[dataset['engine']=='google'].copy()\n",
    "bing_dataset = dataset[dataset['engine']=='bing'].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74ade79e",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Google dataset size: \", google_dataset.shape[0])\n",
    "print(\"Bing dataset size: \", bing_dataset.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ccaa88a",
   "metadata": {},
   "outputs": [],
   "source": [
    "bing_dataset = bing_dataset[:78]\n",
    "print(\"Bing dataset size: \", bing_dataset.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04a9f11f",
   "metadata": {},
   "source": [
    "## Visualize dependencies \n",
    "https://spacy.io/usage/visualizers#dep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35dbd050",
   "metadata": {},
   "outputs": [],
   "source": [
    "options={'compact': True, 'distance': 90}\n",
    "\n",
    "doc = nlp('\"Happy White Woman Pregnant, Black Man. Stock Photo - Image of ...\"')\n",
    "displacy.render(doc, style=\"dep\", options=options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf904e61",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "doc = nlp('\"Happy Woman With Big Smile, Studio White Background ...\"')\n",
    "displacy.render(doc, style=\"dep\", options=options)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9d2e460",
   "metadata": {},
   "source": [
    "## Get dependency and dependency heads for word 'white'\n",
    "https://spacy.io/api/dependencyparser  \n",
    "https://spacy.io/usage/linguistic-features#dependency-parse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26023073",
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to get dependency type for word in text \n",
    "def get_dep_type(text, word):\n",
    "    deps = []\n",
    "    doc = nlp(text)\n",
    "    for token in doc:\n",
    "        if token.text.lower() == word.lower():\n",
    "            deps.append(token.dep_)\n",
    "    return deps\n",
    "\n",
    "# function to get dependency head for word in text \n",
    "def get_dep_head(text, word):\n",
    "    heads = []\n",
    "    doc = nlp(text)\n",
    "    for token in doc:\n",
    "        if token.text.lower() == word.lower():\n",
    "            heads.append(token.head.text.lower())\n",
    "    return heads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f50f0e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "google_dataset['deps'] = google_dataset['title'].apply(get_dep_type, word='white')\n",
    "google_dataset['deps head'] = google_dataset['title'].apply(get_dep_head, word='white')\n",
    "\n",
    "bing_dataset['deps'] = bing_dataset['title'].apply(get_dep_type, word='white')\n",
    "bing_dataset['deps head'] = bing_dataset['title'].apply(get_dep_head, word='white')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "935fad17",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check dependency types for word white in titles\n",
    "deps = list(itertools.chain.from_iterable(google_dataset['deps']))\n",
    "Counter(deps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f270f5cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "deps = list(itertools.chain.from_iterable(bing_dataset['deps']))\n",
    "Counter(deps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "526dcb22",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Count dependency heads for word white in titles \n",
    "bing_heads = list(itertools.chain.from_iterable(bing_dataset['deps head']))\n",
    "bing_cnt = Counter(bing_heads)\n",
    "bing_cnt.most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d68af43",
   "metadata": {},
   "outputs": [],
   "source": [
    "google_heads = list(itertools.chain.from_iterable(google_dataset['deps head']))\n",
    "google_cnt = Counter(google_heads)\n",
    "google_cnt.most_common()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0bbb7cd",
   "metadata": {},
   "source": [
    "### Plot results\n",
    "We will use bar plot to show most common heads."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "705d6e22",
   "metadata": {},
   "outputs": [],
   "source": [
    "cnt = google_cnt + bing_cnt\n",
    "labels = [k for k, v in cnt.most_common(15)]\n",
    "google_values = [google_cnt[lbl] for lbl in labels]\n",
    "bing_values = [bing_cnt[lbl] for lbl in labels]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ba009a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams[\"figure.figsize\"] = (20,10)\n",
    "\n",
    "x = np.arange(len(labels))\n",
    "width = 0.4\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "ax.bar(x-width/2, google_values, width=width, label='google')\n",
    "ax.bar(x+width/2, bing_values, width=width, label='bing')\n",
    "ax.set_xticks(x)\n",
    "ax.set_xticklabels(labels, fontsize=15, rotation=45)\n",
    "ax.legend(fontsize=20)\n",
    "ax.set_title(\"Heads count for word 'white'\", fontsize=15)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98d82ea8",
   "metadata": {},
   "source": [
    "## Check other queries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5eff50b",
   "metadata": {},
   "outputs": [],
   "source": [
    "a_dataset = df[df['query'] == 'happy+asian+woman'].copy()\n",
    "b_dataset = df[df['query'] == 'happy+black+woman'].copy()\n",
    "\n",
    "a_dataset['deps head'] = a_dataset['title'].apply(get_dep_head, word='asian')\n",
    "b_dataset['deps head'] = b_dataset['title'].apply(get_dep_head, word='black')\n",
    "\n",
    "a_heads = list(itertools.chain.from_iterable(a_dataset['deps head']))\n",
    "b_heads = list(itertools.chain.from_iterable(b_dataset['deps head']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4e2ee7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "a_cnt = Counter(a_heads)\n",
    "a_cnt.most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9fceb5f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "b_cnt = Counter(b_heads)\n",
    "b_cnt.most_common()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7a02f41",
   "metadata": {},
   "source": [
    "## TODO:\n",
    "* Check what noun phrases we can find in documents. Use doc.noun_chunks iterator.  \n",
    "https://spacy.io/api/doc#noun_chunks"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}