bias-measurement-GloVe.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "\n",
    "import numpy as np\n",
    "import scipy\n",
    "from scipy.stats.stats import spearmanr\n",
    "from scipy.stats.stats import pearsonr\n",
    "from sklearn.decomposition import PCA\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading ..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Word embedding model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get sure that you have downloaded the provided GloVe word embedding. If you want to create a new set of vectors, you can simply use the tool, originally provided by Stanford NLP."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading completed!\n",
      "Number of word vectors: 197549\n"
     ]
    }
   ],
   "source": [
    "we_model_path = 'word_embeddings/GloVe/vectors.txt'\n",
    "we_vector_size = 300\n",
    "\n",
    "we_vocabs = []\n",
    "we_vectors = []\n",
    "we_biases = []\n",
    "we_ctx_vectors = []\n",
    "we_ctx_biases = []\n",
    "with open(we_model_path) as fr:\n",
    "    for line in fr:\n",
    "        vals = line.strip().split()\n",
    "        we_vocabs.append(vals[0])\n",
    "        we_vectors.append([float(x) for x in vals[1:1+we_vector_size]])\n",
    "        we_biases.append([float(x) for x in vals[1+we_vector_size:2+we_vector_size]][0])\n",
    "        we_ctx_vectors.append([float(x) for x in vals[2+we_vector_size:2+2*we_vector_size]])\n",
    "        we_ctx_biases.append([float(x) for x in vals[2+2*we_vector_size:]][0])\n",
    "        \n",
    "\n",
    "we_vocabs = np.array(we_vocabs)\n",
    "we_vectors = np.array(we_vectors)\n",
    "we_biases = np.array(we_biases)\n",
    "we_ctx_vectors = np.array(we_ctx_vectors)\n",
    "we_ctx_biases = np.array(we_ctx_biases)\n",
    "\n",
    "print ('Loading completed!')\n",
    "print ('Number of word vectors:', len(we_vocabs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "we_vocabs_indextowords = {}\n",
    "for v_i, v in enumerate(we_vocabs):\n",
    "    we_vocabs_indextowords[v] = v_i"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Word lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of jobs: 497\n"
     ]
    }
   ],
   "source": [
    "jobs = []\n",
    "with open('resources/wordlist_occupations.txt') as fr:\n",
    "    for l in fr:\n",
    "        _tuple=l.strip('\\n').split(',')\n",
    "        if _tuple[0] in we_vocabs:\n",
    "            jobs.append(_tuple[0])\n",
    "\n",
    "print (\"Number of jobs:\", len(jobs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of female specific words: 28\n",
      "Number of male specific words: 28\n"
     ]
    }
   ],
   "source": [
    "representative_words_feml = []\n",
    "representative_words_male = []\n",
    "\n",
    "with open('resources/wordlist_genderspecific.txt') as fr:\n",
    "    for l in fr:\n",
    "        _tuple=l.strip('\\n').split(',')\n",
    "        if _tuple[0] in we_vocabs:\n",
    "            if _tuple[1] == 'f':\n",
    "                representative_words_feml.append(_tuple[0])\n",
    "            elif _tuple[1] == 'm':\n",
    "                representative_words_male.append(_tuple[0])\n",
    "        else:\n",
    "            print (\"Missing\", _tuple[0])\n",
    "print (\"Number of female specific words:\", len(representative_words_feml))\n",
    "print (\"Number of male specific words:\", len(representative_words_male))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of pairs of gendered words: 28\n"
     ]
    }
   ],
   "source": [
    "gender_words_direction = []\n",
    "with open('resources/wordpairs_direction.txt') as fr:\n",
    "    for l in fr:\n",
    "        gender_words_direction.append(l.strip().split(','))\n",
    "print (\"Number of pairs of gendered words:\", len(gender_words_direction))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculting gender bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Cosine(vec1, vec2):\n",
    "    return 1 - scipy.spatial.distance.cosine(vec1, vec2)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Smoothed First Order - Average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "context_vectors_feml = []\n",
    "context_vectors_male = []\n",
    "\n",
    "for _w in representative_words_feml:\n",
    "    _w_idx = we_vocabs_indextowords[_w]\n",
    "    _context_vec = we_ctx_vectors[_w_idx]\n",
    "    context_vectors_feml.append(_context_vec)\n",
    "context_vectors_feml = np.array(context_vectors_feml)\n",
    "\n",
    "for _w in representative_words_male:\n",
    "    _w_idx = we_vocabs_indextowords[_w]\n",
    "    _context_vec = we_ctx_vectors[_w_idx]\n",
    "    context_vectors_male.append(_context_vec)\n",
    "context_vectors_male = np.array(context_vectors_male)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Smoothed first-order relation of 'nurse' to female: 2.580334\n",
      "Smoothed first-order relation of 'nurse' to male: 1.507453\n"
     ]
    }
   ],
   "source": [
    "def get_firstorder_average(word, gender_context_vectors):\n",
    "    \n",
    "    _word_vec = we_vectors[we_vocabs_indextowords[word]]\n",
    "\n",
    "    _relations = np.dot(gender_context_vectors, _word_vec)\n",
    "    return np.mean(_relations)\n",
    "\n",
    "word = 'nurse'\n",
    "print (\"Smoothed first-order relation of '%s' to female: %f\" % (word, get_firstorder_average(word, context_vectors_feml)))\n",
    "print (\"Smoothed first-order relation of '%s' to male: %f\" % (word, get_firstorder_average(word, context_vectors_male)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "bias_firstorder_average = {}\n",
    "for _job in jobs:\n",
    "    _bias = get_firstorder_average(_job, context_vectors_feml) - get_firstorder_average(_job, context_vectors_male)\n",
    "    bias_firstorder_average[_job] = _bias\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### High Order - Directional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "_matrix = []\n",
    "for a, b in gender_words_direction:\n",
    "    _a_vec = we_vectors[we_vocabs_indextowords[a]]\n",
    "    _b_vec = we_vectors[we_vocabs_indextowords[b]]\n",
    "    _matrix.append((_a_vec - _b_vec)/2.0)\n",
    "    _matrix.append((_b_vec - _a_vec)/2.0)\n",
    "    \n",
    "_matrix = np.array(_matrix)\n",
    "pca = PCA(n_components = 10)\n",
    "gender_direction_vec = pca.fit(_matrix).components_[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "High-order bias of 'nurse' in respect to female-male direction: 1.026388\n"
     ]
    }
   ],
   "source": [
    "def get_highorder_directional(word, gender_direction_vec):\n",
    "    _word_vec = we_vectors[we_vocabs_indextowords[word]]\n",
    "    return np.dot(_word_vec, gender_direction_vec)\n",
    "\n",
    "word = 'nurse'\n",
    "print (\"High-order bias of '%s' in respect to female-male direction: %f\" % (word, get_highorder_directional(word, gender_direction_vec)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "bias_highorder_directional = {}\n",
    "for _job in jobs:\n",
    "    bias_highorder_directional[_job] = get_highorder_directional(_job, gender_direction_vec)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### High Order - Centriod"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_vector_centroid_feml = np.mean(np.array([we_vectors[we_vocabs_indextowords[_w]] for _w in representative_words_feml]), axis=0)\n",
    "word_vector_centroid_male = np.mean(np.array([we_vectors[we_vocabs_indextowords[_w]] for _w in representative_words_male]), axis=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "High-order relation of 'nurse' to female: 0.505788\n",
      "High-order relation of 'nurse' to male: 0.266729\n"
     ]
    }
   ],
   "source": [
    "def get_highorder_centroid(word, vector_centroid):\n",
    "    _word_vec = we_vectors[we_vocabs_indextowords[word]]\n",
    "    return Cosine(vector_centroid, _word_vec)\n",
    "\n",
    "word = 'nurse'\n",
    "print (\"High-order relation of '%s' to female: %f\" % (word, get_highorder_centroid(word, word_vector_centroid_feml)))\n",
    "print (\"High-order relation of '%s' to male: %f\" % (word, get_highorder_centroid(word, word_vector_centroid_male)))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "bias_highorder_centroid = {}\n",
    "for _job in jobs:\n",
    "    _bias = get_highorder_centroid(_job, word_vector_centroid_feml) - get_highorder_centroid(_job, word_vector_centroid_male)\n",
    "    bias_highorder_centroid[_job] = _bias\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### High Order - Average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_vectors_feml = []\n",
    "word_vectors_male = []\n",
    "\n",
    "for _w in representative_words_feml:\n",
    "    word_vectors_feml.append(we_vectors[we_vocabs_indextowords[_w]])\n",
    "word_vectors_feml = np.array(word_vectors_feml)\n",
    "\n",
    "for _w in representative_words_male:\n",
    "    word_vectors_male.append(we_vectors[we_vocabs_indextowords[_w]])\n",
    "word_vectors_male = np.array(word_vectors_male)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "High-order relation of 'nurse' to female: 0.273086\n",
      "High-order relation of 'nurse' to male: 0.147788\n"
     ]
    }
   ],
   "source": [
    "def get_highorder_average(word, gender_word_vectors):\n",
    "    \n",
    "    _word_vec = we_vectors[we_vocabs_indextowords[word]]\n",
    "    _relations = np.array([Cosine(_gen_vec, _word_vec) for _gen_vec in gender_word_vectors])\n",
    "    \n",
    "    return np.mean(_relations)\n",
    "\n",
    "word = 'nurse'\n",
    "print (\"High-order relation of '%s' to female: %f\" % (word, get_highorder_average(word, word_vectors_feml)))\n",
    "print (\"High-order relation of '%s' to male: %f\" % (word, get_highorder_average(word, word_vectors_male)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "bias_highorder_average = {}\n",
    "for _job in jobs:\n",
    "    _bias = get_highorder_average(_job, word_vectors_feml) - get_highorder_average(_job, word_vectors_male)\n",
    "    bias_highorder_average[_job] = _bias\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Correlation to Occupation Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of data points in WinoBias dataset:  40\n",
      "Number of data points in WinoBias dataset:  96\n"
     ]
    }
   ],
   "source": [
    "\n",
    "winobias_stats = {}\n",
    "with open('resources/occupations_stats_winobias.txt') as fr:\n",
    "    for l in fr:\n",
    "        _tuple=l.strip('\\n').split(',')\n",
    "        winobias_stats[_tuple[0]] = float(_tuple[1])\n",
    "print (\"Number of data points in WinoBias dataset: \", len(winobias_stats))\n",
    "\n",
    "census_stats={}\n",
    "with open('resources/occupations_stats_census.txt') as fr:\n",
    "    for l in fr:\n",
    "        _tuple=l.strip('\\n').split(',')\n",
    "        census_stats[_tuple[0]] = float(_tuple[1])\n",
    "print (\"Number of data points in WinoBias dataset: \", len(census_stats))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Labor Data\n",
      "----------\n",
      "High Order - Directional\n",
      "Spearman 0.511 \t Pearson 0.544\n",
      "High Order - Centroid\n",
      "Spearman 0.577 \t Pearson 0.599\n",
      "High Order - Average\n",
      "Spearman 0.596 \t Pearson 0.610\n",
      "First Order - Average\n",
      "Spearman 0.560 \t Pearson 0.574\n",
      "\n",
      "Census Data\n",
      "----------\n",
      "High Order - Directional\n",
      "Spearman 0.337 \t Pearson 0.445\n",
      "High Order - Centroid\n",
      "Spearman 0.394 \t Pearson 0.507\n",
      "High Order - Average\n",
      "Spearman 0.394 \t Pearson 0.507\n",
      "First Order - Average\n",
      "Spearman 0.423 \t Pearson 0.520\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def calc_correlations(bias_stats, bias_text):\n",
    "    \n",
    "    _list1 = []\n",
    "    _list2 = []\n",
    "    for _job in bias_stats:\n",
    "        _list1.append(bias_stats[_job])\n",
    "        _list2.append(bias_text[_job])\n",
    "        \n",
    "    results = {}\n",
    "    results['spearman'] = abs(spearmanr(_list1, _list2).correlation)\n",
    "    results['pearson'] = abs(pearsonr(_list1, _list2)[0])\n",
    "    \n",
    "    print (\"Spearman %0.3f \\t Pearson %0.3f\" % (results['spearman'], results['pearson']))\n",
    "    \n",
    "    return results\n",
    "    \n",
    "\n",
    "results = {'winobias': {}, 'census': {}}\n",
    "\n",
    "print (\"Labor Data\")\n",
    "print (\"----------\")\n",
    "print ('High Order - Directional')\n",
    "results['winobias']['highorder_directional'] = calc_correlations(winobias_stats, bias_highorder_directional)\n",
    "print ('High Order - Centroid')\n",
    "results['winobias']['highorder_centroid'] = calc_correlations(winobias_stats, bias_highorder_centroid)\n",
    "print ('High Order - Average')\n",
    "results['winobias']['highorder_average'] = calc_correlations(winobias_stats, bias_highorder_average)\n",
    "print ('First Order - Average')\n",
    "results['winobias']['firstorder_average'] = calc_correlations(winobias_stats, bias_firstorder_average)\n",
    "print ()\n",
    "\n",
    "print (\"Census Data\")\n",
    "print (\"----------\")\n",
    "print ('High Order - Directional')\n",
    "results['census']['highorder_directional'] = calc_correlations(census_stats, bias_highorder_directional)\n",
    "print ('High Order - Centroid')\n",
    "results['census']['highorder_centroid'] = calc_correlations(census_stats, bias_highorder_centroid)\n",
    "print ('High Order - Average')\n",
    "results['census']['highorder_average'] = calc_correlations(census_stats, bias_highorder_average)\n",
    "print ('First Order - Average')\n",
    "results['census']['firstorder_average'] = calc_correlations(census_stats, bias_firstorder_average)\n",
    "print ()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}