wide_deep_fea2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import shutil\n",
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add a parser for the program, specify column names for the data and assign a default value for each column if the data is not available in that column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "_StoreAction(option_strings=['--test_data'], dest='test_data', nargs=None, const=None, default='./valid_phase2.csv', type=<class 'str'>, choices=None, help='Path to the test data.', metavar=None)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "phase = 2\n",
    "\n",
    "column_default = [\n",
    "    # phase2\n",
    "    ('', 0), \n",
    "\t('msno','missing_msno'),\n",
    "\t('song_id','missing_song_id'),\n",
    "\t('source_system_tab','missing_tab'),\n",
    "    ('source_screen_name','missing_screen'),\n",
    "\t('source_type','missing_source_type'),\n",
    "    ('target',''),\n",
    "\t('song_length',241812.0),\n",
    "\t('genre_ids','empty'),\n",
    "\t('artist_name','empty'),\n",
    "    ('composer','empty'),\n",
    "\t('lyricist','empty'),\n",
    "\t('language',-1.0),\n",
    "    ('short_song',0.0),\n",
    "\t('mean_length_distance',54.0),\n",
    "\t('city',-1),\n",
    "\t('bd',18),\n",
    "\t('gender','empty'),\n",
    "    ('registered_via',0),\n",
    "\t('registration_init_time',''),\n",
    "\t('expiration_date',0),\n",
    "\t('membership_days',0),\n",
    "    ('registration_year',-1),\n",
    "\t('registration_month',-1),\n",
    "\t('registration_date',-1),\n",
    "\t('expiration_year',-1),\n",
    "    ('expiration_month',-1),\n",
    "\t('song_name','empty'),\n",
    "\t('country_code','empty'),\n",
    "    ('issuer','empty'),\n",
    "\t('issue_year',0.0),\n",
    "\t('unique_id','empty'),\n",
    "\t('genre_ids_count',0),\n",
    "\t('lyricists_count',0),\n",
    "\t('composer_count',0),\n",
    "\t('artist_count',0),\n",
    "\t('is_featured_artist',0),\n",
    "\t('is_featured_song',0),\n",
    "\t('is_featured',0),\n",
    "\t('feat_artist','empty'),\n",
    "\t('main_artist','empty'),\n",
    "\t('feat_song_name','empty'),\n",
    "\t('main_song_name','empty'),\n",
    "\t('is_remix_artist',0),\n",
    "\t('is_remix_song',0),\n",
    "\t('is_remix',0),\n",
    "\t('is_live_artist',0),\n",
    "\t('is_live_song',0),\n",
    "\t('is_live',0),\n",
    "\t('is_acoustic_artist',0),\n",
    "\t('is_acoustic_song',0),\n",
    "\t('is_acoustic',0),\n",
    "\t('is_instrumental_artist',0),\n",
    "\t('is_instrumental_song',0),\n",
    "\t('is_instrumental',0),\n",
    "\t('artist_is_composer',0),\n",
    "\t('artist_is_composer_is_lyricist',0),\n",
    "\t('song_lang_magic',0),\n",
    "\t('count_song_played',425),\n",
    "\t('count_artist_played',406),\n",
    "\t('count_composer_played',171),\n",
    "\t('count_lyricist_played',226),\n",
    "\t('count_member_action',848),\n",
    "\t('member_action_per_day',0.0)\n",
    "]\n",
    "\n",
    "_CSV_COLUMNS = []\n",
    "_CSV_COLUMN_DEFAULTS = []\n",
    "\n",
    "# print(column_default)\n",
    "\n",
    "for key in column_default:\n",
    "    _CSV_COLUMNS.append(key[0])\n",
    "    _CSV_COLUMN_DEFAULTS.append([key[1]])\n",
    "\n",
    "#print(_CSV_COLUMNS)\n",
    "parser = argparse.ArgumentParser()\n",
    "\n",
    "parser.add_argument(\n",
    "    '--model_dir', type=str, default='/tmp/census_model',\n",
    "    help='Base directory for the model.')\n",
    "\n",
    "parser.add_argument(\n",
    "    '--model_type', type=str, default='wide_deep',\n",
    "    help=\"Valid model types: {'wide', 'deep', 'wide_deep'}.\")\n",
    "\n",
    "parser.add_argument(\n",
    "    '--train_epochs', type=int, default=40, help='Number of training epochs.')\n",
    "\n",
    "parser.add_argument(\n",
    "    '--epochs_per_eval', type=int, default=2,\n",
    "    help='The number of training epochs to run between evaluations.')\n",
    "\n",
    "parser.add_argument(\n",
    "    '--batch_size', type=int, default=512, help='Number of examples per batch.')\n",
    "\n",
    "parser.add_argument(\n",
    "    '--train_data', type=str, default='./train_phase' + str(phase) + '.csv',\n",
    "    help='Path to the training data.')\n",
    "\n",
    "parser.add_argument(\n",
    "    '--test_data', type=str, default='./valid_phase' + str(phase) + '.csv',\n",
    "    help='Path to the test data.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split the data into a training set and a validation set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "_NUM_EXAMPLES = {\n",
    "    'train': 5901935 + 1474484 + 1000,\n",
    "    'validation': 0,#1474484 + 1000,\n",
    "}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Give parameters for hashing categorical features. Create a crossed feature with several categorical features. Feed these features into the wide neural network. Embed several categorical features to put them and other numerical features in the deep neural network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_model_columns():\n",
    "  \"\"\"Builds a set of wide and deep feature columns.\"\"\"\n",
    "  # Continuous columns\n",
    "  categorical_features = {}\n",
    "  numerical_features = {}\n",
    "  for key in column_default:\n",
    "    if type(key[1])!=str:\n",
    "        numerical_features[key[0]] = tf.feature_column.numeric_column(key[0])\n",
    "    elif key[0] != 'target':\n",
    "        categorical_features[key[0]] = tf.feature_column.categorical_column_with_hash_bucket(key[0], hash_bucket_size=3200000)\n",
    "\n",
    "  categorical_features['gender'] = tf.feature_column.categorical_column_with_vocabulary_list(\n",
    "      'gender', ['male', 'female'])\n",
    "        \n",
    "\n",
    "  # Transformations.\n",
    "  age_buckets = tf.feature_column.bucketized_column(\n",
    "      numerical_features['bd'], boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65])\n",
    "  song_length_buckets = tf.feature_column.bucketized_column(\n",
    "      numerical_features['song_length'], boundaries=[160000, 320000, 480000, 640000])\n",
    "\n",
    "  # Wide columns and deep columns.\n",
    "  base_columns = []\n",
    "  for key in categorical_features:\n",
    "    base_columns.append(categorical_features[key])\n",
    "\n",
    "  crossed_columns = [\n",
    "      tf.feature_column.crossed_column(\n",
    "          ['artist_name', 'composer', 'lyricist'], hash_bucket_size=10000),\n",
    "  ]\n",
    "\n",
    "  wide_columns = base_columns + crossed_columns\n",
    "\n",
    "  deep_columns = []\n",
    "  for key in numerical_features:\n",
    "    deep_columns.append(numerical_features[key])\n",
    "  for key in categorical_features:\n",
    "    if (key=='song_length'):\n",
    "      deep_columns.append(tf.feature_column.numeric_column('song_length', normalizer_fn=lambda x: (x - 241812.0) / 67351))\n",
    "    else:\n",
    "      deep_columns.append(tf.feature_column.embedding_column(categorical_features[key], 24))\n",
    "\n",
    "  return wide_columns, deep_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Specify number of hidden layers and hidden units in the deep neural network. Return a wide network estimator, a deep network estimator or a wide and deep network estimator according to the model type provided in the parser."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_estimator(model_dir, model_type):\n",
    "  \"\"\"Build an estimator appropriate for the given model type.\"\"\"\n",
    "  wide_columns, deep_columns = build_model_columns()\n",
    "  hidden_units = [128, 128, 64, 64, 32, 32]#[100, 75, 50, 25]\n",
    "\n",
    "  # Create a tf.estimator.RunConfig to ensure the model is run on CPU, which\n",
    "  # trains faster than GPU for this model.\n",
    "  run_config = tf.estimator.RunConfig().replace(\n",
    "      session_config=tf.ConfigProto(device_count={'GPU': 0}))\n",
    "  \n",
    "  #config.intra_op_parallelism_threads = 44\n",
    "  #config.intra_op_parallelism_threads = 44\n",
    "    \n",
    "  if model_type == 'wide':\n",
    "    return tf.estimator.LinearClassifier(\n",
    "        model_dir=model_dir,\n",
    "        feature_columns=wide_columns,\n",
    "        config=run_config)\n",
    "  elif model_type == 'deep':\n",
    "    return tf.estimator.DNNClassifier(\n",
    "        model_dir=model_dir,\n",
    "        feature_columns=deep_columns,\n",
    "        hidden_units=hidden_units,\n",
    "        dropout=0.05,\n",
    "        config=run_config)\n",
    "  else:\n",
    "    return tf.estimator.DNNLinearCombinedClassifier(\n",
    "        model_dir=model_dir,\n",
    "        linear_feature_columns=wide_columns,\n",
    "        dnn_feature_columns=deep_columns,\n",
    "        dnn_hidden_units=hidden_units,\n",
    "        dnn_dropout=0.05,\n",
    "        config=run_config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generate a function that produces input for the tensorflow graph. The function will parse the csv file, do the shuffling, extract the label, and feed the features to the estimator."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def input_fn(data_file, num_epochs, shuffle, batch_size):\n",
    "  \"\"\"Generate an input function for the Estimator.\"\"\"\n",
    "  assert tf.gfile.Exists(data_file), (\n",
    "      '%s not found. Please make sure you have either run data_download.py or '\n",
    "      'set both arguments --train_data and --test_data.' % data_file)\n",
    "\n",
    "  def parse_csv(value):\n",
    "    print('Parsing', data_file)\n",
    "    columns = tf.decode_csv(value, field_delim = ',', record_defaults=_CSV_COLUMN_DEFAULTS)\n",
    "    features = dict(zip(_CSV_COLUMNS, columns))\n",
    "    labels = features.pop('target')\n",
    "    return features, tf.equal(labels, '1')\n",
    "\n",
    "  # Extract lines from input files using the Dataset API.\n",
    "  dataset = tf.data.TextLineDataset(data_file)\n",
    "  dataset = dataset.skip(1)\n",
    "  if shuffle:\n",
    "    dataset = dataset.shuffle(buffer_size=_NUM_EXAMPLES['train'])\n",
    "\n",
    "  dataset = dataset.map(parse_csv, num_parallel_calls=5) #skip(7376419)\n",
    "\n",
    "  # We call repeat after shuffling, rather than before, to prevent separate\n",
    "  # epochs from blending together.\n",
    "  dataset = dataset.repeat(num_epochs)\n",
    "  dataset = dataset.batch(batch_size)\n",
    "\n",
    "  iterator = dataset.make_one_shot_iterator()\n",
    "  features, labels = iterator.get_next()\n",
    "  return features, labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A logger that helps to keep track of the training process."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "logger = logging.getLogger()\n",
    "fhandler = logging.FileHandler(filename='mylog.log', mode='a')\n",
    "formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')\n",
    "fhandler.setFormatter(formatter)\n",
    "logger.addHandler(fhandler)\n",
    "logger.setLevel(logging.DEBUG)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train the wide and deep neural network for epochs specified in the parser, and then predict the labels for the test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def predict(model, epoch, phase):\n",
    "  fea_str = \",id,msno,song_id,source_system_tab,source_screen_name,source_type,song_length,genre_ids,artist_name,composer,lyricist,language,short_song,mean_length_distance,city,bd,gender,registered_via,registration_init_time,expiration_date,membership_days,registration_year,registration_month,registration_date,expiration_year,expiration_month,song_name,country_code,issuer,issue_year,unique_id,genre_ids_count,lyricists_count,composer_count,artist_count,is_featured_artist,is_featured_song,is_featured,feat_artist,main_artist,feat_song_name,main_song_name,is_remix_artist,is_remix_song,is_remix,is_live_artist,is_live_song,is_live,is_acoustic_artist,is_acoustic_song,is_acoustic,is_instrumental_artist,is_instrumental_song,is_instrumental,artist_is_composer,artist_is_composer_is_lyricist,song_lang_magic,count_song_played,count_artist_played,count_composer_played,count_lyricist_played,count_member_action,member_action_per_day\"\n",
    "  column_names2 = fea_str.split(\",\")\n",
    "\n",
    "  def conv(val):  \n",
    "    try:\n",
    "        return np.float(val)\n",
    "    except:        \n",
    "        return np.float64(0)\n",
    "\n",
    "  def conv1(val):  \n",
    "    try:\n",
    "        return np.float(val)\n",
    "    except:        \n",
    "        return np.float64(-1.0)\n",
    "\n",
    "  df = pd.read_csv('./test_phase' + str(phase) + '.csv', index_col=False, names=column_names2,\n",
    "                            skip_blank_lines=True, keep_default_na=False, skiprows=1,\n",
    "                            converters={'language':conv1, 'mean_length_distance': conv1, \n",
    "                                        'issue_year':conv1, 'song_length': conv, 'short_song':conv1})\n",
    "  predictions = list(model.predict(input_fn=tf.estimator.inputs.pandas_input_fn(\n",
    "    x=df, num_epochs=1,shuffle=False)))\n",
    "  res = pd.DataFrame(predictions)\n",
    "  #temp = model\n",
    "  #temp2 = predictions\n",
    "  res.to_csv('pred.csv', sep = ',', header = False)\n",
    "  column_names3=['id', 'label', 'score', 'target', 's2', 's']\n",
    "  def conv3(val):  \n",
    "    return np.int(val[1])\n",
    "  def conv4(val):\n",
    "    return np.float(val[1:-1])\n",
    "  census_pred = pd.read_csv('./pred.csv', index_col=False, names=column_names3,\n",
    "                          skip_blank_lines=True, keep_default_na=False, converters={'target': conv4})\n",
    "  header=['id', 'target']\n",
    "  census_pred.to_csv('output_float_phase' + str(phase) + '_epoch_' + str(epoch) + '.csv', index=False, columns = header)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "temp = 0\n",
    "temp2 = 0\n",
    "model = None\n",
    "def main(unused_argv):\n",
    "  global temp\n",
    "  global temp2\n",
    "  global model\n",
    "  # Clean up the model directory if present\n",
    "  shutil.rmtree(FLAGS.model_dir, ignore_errors=True)\n",
    "  model = build_estimator(FLAGS.model_dir, FLAGS.model_type)\n",
    "\n",
    "  # Train and evaluate the model every `FLAGS.epochs_per_eval` epochs.\n",
    "  for n in range(21):#range(FLAGS.train_epochs // FLAGS.epochs_per_eval):\n",
    "    model.train(input_fn=lambda: input_fn(\n",
    "        FLAGS.train_data, FLAGS.epochs_per_eval, True, FLAGS.batch_size))\n",
    "\n",
    "    if n % 5 == 0:\n",
    "      results = model.evaluate(input_fn=lambda: input_fn(\n",
    "          FLAGS.test_data, 1, False, FLAGS.batch_size))\n",
    "      for key in sorted(results):\n",
    "        print('%s: %s' % (key, results[key]))\n",
    "\n",
    "      # Display evaluation metrics\n",
    "      print('Results at epoch', (n + 1) * FLAGS.epochs_per_eval)\n",
    "      print('-' * 60)\n",
    "      predict(model, n, phase)\n",
    "    \n",
    "\n",
    "if __name__ == '__main__':\n",
    "  tf.logging.set_verbosity(tf.logging.INFO)\n",
    "  FLAGS, unparsed = parser.parse_known_args()\n",
    "  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed + [temp])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(\"hi\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract the labels from the prediction result to generate submission file output.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "column_names3=['id', 'label', 'score', 'target', 's2', 's']\n",
    "def conv3(val):  \n",
    "    return np.int(val[1])\n",
    "def conv4(val):\n",
    "    return np.float(val[1:-1])\n",
    "census_pred = pd.read_csv('./pred.csv', index_col=False, names=column_names3,\n",
    "                          skip_blank_lines=True, keep_default_na=False, converters={'target': conv4})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "census_pred['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "header=['id', 'target']\n",
    "census_pred.to_csv('output_float.csv', index=False, columns = header)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "column_names2 = [\n",
    "    '','id','msno','song_id','source_system_tab','source_screen_name','source_type',\n",
    "      'song_length','genre_ids','artist_name','composer','lyricist','language','short_song',\n",
    "      'mean_length_distance','city','bd','gender','registered_via','registration_init_time',\n",
    "      'expiration_date','membership_days','registration_year','registration_month',\n",
    "      'registration_date','expiration_year','expiration_month','song_name','country_code',\n",
    "      'issuer','issue_year','unique_id'\n",
    "  ]\n",
    "\n",
    "def conv(val):  \n",
    "    try:\n",
    "        return np.float(val)\n",
    "    except:        \n",
    "        return np.float64(0)\n",
    "\n",
    "def conv1(val):  \n",
    "    try:\n",
    "        return np.float(val)\n",
    "    except:        \n",
    "        return np.float64(-1.0)\n",
    "\n",
    "df = pd.read_csv('./test_phase1.csv', index_col=False, names=column_names2,\n",
    "                            skip_blank_lines=True, keep_default_na=False, skiprows=1,\n",
    "                            converters={'language':conv1, 'mean_length_distance': conv1, \n",
    "                                        'issue_year':conv1, 'song_length': conv, 'short_song':conv1})\n",
    "predictions = list(model.predict(input_fn=tf.estimator.inputs.pandas_input_fn(\n",
    "  x=df, num_epochs=1,shuffle=False)))\n",
    "res = pd.DataFrame(predictions)\n",
    "temp = model\n",
    "temp2 = predictions\n",
    "res.to_csv('pred.csv', sep = ',', header = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "type(df['song_length'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}