playlister.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 367
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "D0LlOQ_U5YOE",
    "outputId": "b982c01f-1812-47b3-da05-6edc36dfadb7"
   },
   "outputs": [],
   "source": [
    "import pandas\n",
    "import requests\n",
    "import spotipy\n",
    "import string\n",
    "import re\n",
    "\n",
    "from spotipy.oauth2 import SpotifyClientCredentials\n",
    "from genderize import Genderize\n",
    "from numpy import nan\n",
    "\n",
    "# Authenticate from offline file\n",
    "with open('vars.env', 'r') as f:\n",
    "    my_id = f.readline().rstrip('\\n')\n",
    "    my_secret = f.readline().rstrip('\\n')\n",
    "    playlistURI = f.readline().rstrip('\\n')\n",
    "    owner = f.readline().rstrip('\\n')\n",
    "\n",
    "# Get playlist from Spotify \n",
    "client_credentials_manager = SpotifyClientCredentials(client_id=my_id, client_secret=my_secret)\n",
    "sp = spotipy.Spotify(client_credentials_manager=client_credentials_manager)\n",
    "playlistID = playlistURI[-22:]\n",
    "results = sp.user_playlist(owner, playlist_id=playlistID, fields='tracks, next')['tracks']\n",
    "songs =  results['items']\n",
    "\n",
    "# Page through songs in playlist\n",
    "while results['next']:\n",
    "    results = sp.next(results)\n",
    "    songs.extend(results['items'])\n",
    "\n",
    "# Work out who added each song and get their display names\n",
    "idSet = {track['added_by']['id'] for track in songs}    \n",
    "us = {(id, sp.user(id)['display_name']) for id in idSet}\n",
    "us = dict(us)\n",
    "us['rmb101'] = 'rmb101'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 201
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "yGI19pocFr9C",
    "outputId": "8b6dd1a0-cfff-4f73-97bf-f729e8eef7c2"
   },
   "outputs": [],
   "source": [
    "# Make the dataframe by pulling out the song name, artist and person who added it\n",
    "table = [(track['track']['name'], track['track']['artists'][0]['name'], us[track['added_by']['id']]) \n",
    "         for track in songs]\n",
    "df = pandas.DataFrame(table, columns = ['Song', 'Artist', 'Added by'])\n",
    "\n",
    "# Get rid of anything after a hyphen, as this is all '2012 remastered' kinda text\n",
    "# also remove all apostrophe s and digits\n",
    "df['Text'] = df['Song'].str.lower()\n",
    "df['Text'].replace('\\d| - .*|\\'s','', regex = True, inplace = True)\n",
    "\n",
    "# set up regex to kill off remaining punctuation\n",
    "punclist = string.punctuation\n",
    "punc = re.compile('[%s]' % re.escape(punclist))\n",
    "\n",
    "# bring in list of common words (ie not names) to remove\n",
    "# also a manual list of words I don't want to pick up as names\n",
    "words = pandas.read_csv('top5000.csv')\n",
    "wordlist = words['Top words'].tolist()\n",
    "wordlist.remove('sue')\n",
    "blocklist = ['rigby', 'marmalade', 'runaround', 'dont', 'hasty', 'mambo', 'holland', 'speedy', 'lind', 'stars', 'o', 'ode', 'punk', 'rocker']\n",
    "wordlist.extend(blocklist)\n",
    "\n",
    "# function to remove the punctuation and common names\n",
    "# returns as list because I'm going to gender assign each word\n",
    "# names set is so I don't need to call genderize on the same name twice\n",
    "nameSet = set()\n",
    "\n",
    "def namesOnly(row):\n",
    "    text = punc.sub('', row['Text']).split()\n",
    "    newTitle = [word for word in text if word not in wordlist]\n",
    "    global nameSet\n",
    "    for word in newTitle:\n",
    "        nameSet.add(word)\n",
    "    return newTitle \n",
    "\n",
    "df['ShortTitle'] = df.apply(namesOnly, axis = 1)\n",
    "\n",
    "# Now add names from artists to the name set and tidy them the same way\n",
    "# Just grab first word from each artist and check if it's a first name\n",
    "df['FirstName'] = df['Artist'].str.lower().str.split(n = 1).str[0]\n",
    "df['FirstName'].replace('\\d| - .*|\\'s','', regex = True, inplace = True)\n",
    "\n",
    "for name in df['FirstName'].unique():\n",
    "    text = punc.sub('', name)\n",
    "    if name not in wordlist:\n",
    "        nameSet.add(name)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Send the whole list off names off to the Genderize API in one go\n",
    "nameScores = Genderize().get(nameSet)\n",
    "genders = {}\n",
    "for name in nameScores:\n",
    "    if name['gender']:\n",
    "        if name['probability'] >= 0.7:\n",
    "            genders[name['name']] = name['gender']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "eA5YHg5_LsWl",
    "outputId": "72be97b3-567a-477c-9763-41c8a8a93317",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Manually add gender for songs it can't guess\n",
    "manual = {'chun-li': 'female', 'lady marmalade': 'female', \"l'autre valse d'amélie\": 'female',\n",
    " 'mr. bojangles': 'male','i loves you porgy': 'male', 'rihanna': 'female',\n",
    " 'dear rose': 'female', 'van gogh ear': 'male', \"rockin' robin\": 'female',\n",
    " 'gurdjieff daughter': 'female', 'seigfried': 'male', 'saint dymphna': 'female'}\n",
    "\n",
    "# Function to check words in song titles against Genderize list\n",
    "def identifySong(name):\n",
    "    guesses = set()\n",
    "    for word in name['ShortTitle']:\n",
    "        if word in genders:\n",
    "            guesses.add(genders[word])\n",
    "    if len(set(guesses)) == 1:\n",
    "        name['Type'] = 'solo'\n",
    "        return guesses.pop()\n",
    "    elif not guesses:\n",
    "        try:\n",
    "            return manual[name['Text']]\n",
    "        except:\n",
    "            return ''\n",
    "    else:\n",
    "        return guesses\n",
    "\n",
    "# Simpler function to get genders for artists from the Genderize list\n",
    "def identifyArtist(name):\n",
    "    guesses = set()\n",
    "    if name['FirstName'] in genders:\n",
    "        return genders[name['FirstName']]\n",
    "    else:\n",
    "        return ''\n",
    "\n",
    "# Apply the functions\n",
    "df['SongGender'] = df.apply(identifySong, axis = 1)\n",
    "df['ArtistGender'] = df.apply(identifyArtist, axis = 1)\n",
    "\n",
    "# Label artists identified so far as solo (this is clumsy)\n",
    "df['Type'] = ''\n",
    "df.Type[df.ArtistGender!=''] = 'solo'\n",
    "\n",
    "# Use a local csv of genders for bands\n",
    "unknowns = pandas.read_csv('Unknowns.csv')\n",
    "\n",
    "# Join the csv up with the main table\n",
    "# There's probably a neater way to do this\n",
    "merged = df.merge(unknowns, 'left', on = 'Artist')\n",
    "merged = merged.replace('', nan)\n",
    "merged['Type'] = merged['Type_x'].combine_first(merged['Type_y'])\n",
    "merged['ArtistGender'] = merged['ArtistGender_x'].combine_first(merged['ArtistGender_y'])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Pick the relevant columns and export results\n",
    "results = merged[['Song', 'Artist', 'Added by', 'SongGender', 'ArtistGender', 'Type']]\n",
    "results.to_csv('SongGenders.csv', index = False)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Playlister",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}