L16qs.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"L16qs.ipynb","provenance":[],"collapsed_sections":[]},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.8"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","metadata":{"id":"OydShklHHtdh"},"source":["This table lists the one-word names of a collection of\n","occupations, along with what fraction of people with\n","that job are female.\n","\n","Jobs and Percent Female\n","* secretary 94.0 \n","* typist 85.1\n","* phlebotomist 75.0 \n","* telemarketer 65.3 \n","* packer 54.5 \n","* appraiser 45.3\n","* dentist 35.7\n","* rancher 25.8\n","* announcer 17.7\n","* firefighter 5.1"]},{"cell_type":"markdown","metadata":{"id":"t4GDNzTYIqso"},"source":["Download GloVe vectors we will be using."]},{"cell_type":"code","metadata":{"id":"sCjJULtUqjAe","executionInfo":{"status":"ok","timestamp":1602708060059,"user_tz":240,"elapsed":392435,"user":{"displayName":"Michael Littman","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1OI1zp1NevMSZ87UN47k8bTZ--Ijr4v3ORV33Vg=s64","userId":"06751891446459829367"}},"outputId":"7cdc99b9-7988-4012-b1ea-1841956d7c0c","colab":{"base_uri":"https://localhost:8080/","height":357}},"source":["!wget http://nlp.stanford.edu/data/glove.6B.zip"],"execution_count":1,"outputs":[{"output_type":"stream","text":["--2020-10-14 20:34:27--  http://nlp.stanford.edu/data/glove.6B.zip\n","Resolving nlp.stanford.edu (nlp.stanford.edu)... 171.64.67.140\n","Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:80... connected.\n","HTTP request sent, awaiting response... 302 Found\n","Location: https://nlp.stanford.edu/data/glove.6B.zip [following]\n","--2020-10-14 20:34:28--  https://nlp.stanford.edu/data/glove.6B.zip\n","Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:443... connected.\n","HTTP request sent, awaiting response... 301 Moved Permanently\n","Location: http://downloads.cs.stanford.edu/nlp/data/glove.6B.zip [following]\n","--2020-10-14 20:34:28--  http://downloads.cs.stanford.edu/nlp/data/glove.6B.zip\n","Resolving downloads.cs.stanford.edu (downloads.cs.stanford.edu)... 171.64.64.22\n","Connecting to downloads.cs.stanford.edu (downloads.cs.stanford.edu)|171.64.64.22|:80... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 862182613 (822M) [application/zip]\n","Saving to: ‘glove.6B.zip’\n","\n","glove.6B.zip        100%[===================>] 822.24M  1.75MB/s    in 6m 30s  \n","\n","2020-10-14 20:40:59 (2.11 MB/s) - ‘glove.6B.zip’ saved [862182613/862182613]\n","\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"GZsutIRzJCzg"},"source":["Unzip the GloVe file we downloaded."]},{"cell_type":"code","metadata":{"id":"5j_YyS37qv7m","executionInfo":{"status":"ok","timestamp":1602708098705,"user_tz":240,"elapsed":23825,"user":{"displayName":"Michael Littman","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1OI1zp1NevMSZ87UN47k8bTZ--Ijr4v3ORV33Vg=s64","userId":"06751891446459829367"}},"outputId":"9774a7bf-4e99-47b7-866b-eaf6074a8f63","colab":{"base_uri":"https://localhost:8080/","height":102}},"source":["!unzip glove.6B.zip"],"execution_count":2,"outputs":[{"output_type":"stream","text":["Archive:  glove.6B.zip\n","  inflating: glove.6B.50d.txt        \n","  inflating: glove.6B.100d.txt       \n","  inflating: glove.6B.200d.txt       \n","  inflating: glove.6B.300d.txt       \n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"lmi-6INoJHbv"},"source":["Next, we load the GloVe vectors."]},{"cell_type":"code","metadata":{"id":"siJVA4bpyPY8","executionInfo":{"status":"ok","timestamp":1602709074090,"user_tz":240,"elapsed":10104,"user":{"displayName":"Michael Littman","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1OI1zp1NevMSZ87UN47k8bTZ--Ijr4v3ORV33Vg=s64","userId":"06751891446459829367"}},"outputId":"5947f182-cac8-4924-b9da-5c01d2ef57c4","colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["import numpy as np\n","\n","embeddings_index = {}\n","f = open('glove.6B.100d.txt')\n","for line in f:\n","    values = line.split()\n","    word = values[0]\n","    coefs = np.asarray(values[1:], dtype='float32')\n","    embeddings_index[word] = coefs\n","f.close()\n","\n","print('Found %s word vectors.' % len(embeddings_index))"],"execution_count":49,"outputs":[{"output_type":"stream","text":["Found 400000 word vectors.\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Hjqy4f3EebpG","executionInfo":{"status":"ok","timestamp":1602709076649,"user_tz":240,"elapsed":516,"user":{"displayName":"Michael Littman","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1OI1zp1NevMSZ87UN47k8bTZ--Ijr4v3ORV33Vg=s64","userId":"06751891446459829367"}}},"source":["jobs = [\"secretary\", \"typist\", \"phlebotomist\", \"telemarketer\", \"packer\", \n","        \"appraiser\", \"dentist\", \"rancher\", \"announcer\", \"firefighter\"]   \n","femrates = [94.0,  85.1,  75.0,  65.3,  54.5,  45.3,  35.7,  25.8,  17.7,  5.1]\n","\n","Aset = [\"female\", \"woman\", \"girl\", \"sister\", \"she\", \"her\", \"hers\", \"daughter\"]\n","Bset = [\"male\", \"man\", \"boy\", \"brother\", \"he\", \"him\", \"his\", \"son\"]\n","\n","def cos(a,b):\n","  return((embeddings_index[a] @ embeddings_index[b])\n","         / np.sqrt(embeddings_index[a] @ embeddings_index[a]) \n","         / np.sqrt(embeddings_index[b] @ embeddings_index[b]))\n","\n","def s(A,B,w):\n","  return((np.mean([cos(w,a) for a in A])-np.mean([cos(w,b) for b in B]))\n","          / np.std([cos(w,x) for x in A+B]))"],"execution_count":50,"outputs":[]},{"cell_type":"code","metadata":{"id":"6jH3XzsmUmE2","executionInfo":{"status":"ok","timestamp":1602709078225,"user_tz":240,"elapsed":532,"user":{"displayName":"Michael Littman","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1OI1zp1NevMSZ87UN47k8bTZ--Ijr4v3ORV33Vg=s64","userId":"06751891446459829367"}},"outputId":"93d4e9f7-6edb-4051-ae87-f53530133f9d","colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["#  50 .29\n","# 100 .29\n","# 200 .44\n","# 300 .41\n","\n","\n","np.corrcoef([s(Aset,Bset,w) for w in jobs],femrates)[0,1]"],"execution_count":51,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.29395930243234547"]},"metadata":{"tags":[]},"execution_count":51}]},{"cell_type":"code","metadata":{"id":"df5zOhEHUlxN","executionInfo":{"status":"ok","timestamp":1602708338817,"user_tz":240,"elapsed":476,"user":{"displayName":"Michael Littman","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1OI1zp1NevMSZ87UN47k8bTZ--Ijr4v3ORV33Vg=s64","userId":"06751891446459829367"}},"outputId":"d147236d-a635-4aa5-e3f7-99119eee95c7","colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["cos(\"phlebotomist\", \"daughter\")"],"execution_count":17,"outputs":[{"output_type":"execute_result","data":{"text/plain":["-0.20062943"]},"metadata":{"tags":[]},"execution_count":17}]},{"cell_type":"code","metadata":{"id":"t8ct2GDWUl8k","executionInfo":{"status":"ok","timestamp":1602708342296,"user_tz":240,"elapsed":637,"user":{"displayName":"Michael Littman","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1OI1zp1NevMSZ87UN47k8bTZ--Ijr4v3ORV33Vg=s64","userId":"06751891446459829367"}},"outputId":"0815af4e-d073-40e8-cd2b-f24e37154a77","colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["cos(\"sister\", \"daughter\")"],"execution_count":18,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.8687514"]},"metadata":{"tags":[]},"execution_count":18}]},{"cell_type":"code","metadata":{"id":"271PMNTNUmA5","executionInfo":{"status":"ok","timestamp":1602708344597,"user_tz":240,"elapsed":419,"user":{"displayName":"Michael Littman","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1OI1zp1NevMSZ87UN47k8bTZ--Ijr4v3ORV33Vg=s64","userId":"06751891446459829367"}},"outputId":"ace4bfd6-7296-4398-f3ef-b9facb6509d3","colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["cos(\"daughter\", \"daughter\")"],"execution_count":19,"outputs":[{"output_type":"execute_result","data":{"text/plain":["1.0000001"]},"metadata":{"tags":[]},"execution_count":19}]}]}