model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os, time , sys\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib.rnn import LSTMCell\n",
    "from tensorflow.contrib.crf import crf_log_likelihood\n",
    "from tensorflow.contrib.crf import viterbi_decode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from data import pad_sequences, batch_yield\n",
    "from utils import get_logger\n",
    "from eval import conlleval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class BiLSTM_CRF(object):\n",
    "    def __init__(self, batch_size, epoch_num, hidden_dim, embeddings,\n",
    "                dropout_keep, optimizer, lr, clip_grad,\n",
    "                tag2label, vocab, shuffle,\n",
    "                model_path, summary_path, log_path, result_path,\n",
    "                CRF=True, update_embedding=True):\n",
    "        self.batch_size = batch_size\n",
    "        self.epoch_num = epoch_num\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.embeddings = embeddings\n",
    "        self.dropout_keep_prob = dropout_keep\n",
    "        self.optimizer = optimizer\n",
    "        self.lr = lr\n",
    "        self.clip_grad = clip_grad\n",
    "        self.tag2label = tag2label\n",
    "        self.num_tags = len(tag2label)\n",
    "        self.vocab = vocab\n",
    "        self.shuffle = shuffle\n",
    "        self.model_path  = model_path\n",
    "        self.summary_path = summary_path\n",
    "        self.logger = get_logger(log_path)\n",
    "        self.result_path = result_path\n",
    "        self.CRF = CRF\n",
    "        self.update_embedding = update_embedding\n",
    "        \n",
    "    def build_graph(self):\n",
    "        self.add_placeholders_op()\n",
    "        self.lookup_layer_op()\n",
    "        self.biLSTM_layer_op()\n",
    "        self.softmax_pred_op()\n",
    "        self.loss_op()\n",
    "        self.trainstep_op()\n",
    "        self.init_op()\n",
    "    \n",
    "    def add_placeholders_op(self):\n",
    "        self.word_ids = tf.placeholder(tf.int32, shape=[None, None], name=\"word_ids\")\n",
    "        self.labels = tf.placeholder(tf.int32, shape=[None, None], name=\"label\")\n",
    "        self.sequence_lengths = tf.placeholder(tf.int32, shape=[None], name=\"sequence_lengths\")\n",
    "        self.dropout_pl = tf.placeholder(tf.float32, shape=[], name=\"dropout\")\n",
    "        self.lr_pl = tf.placeholder(tf.float32, shape=[], name=\"lr\")\n",
    "        \n",
    "    def lookup_layer_op(self):\n",
    "        with tf.variable_scope(\"words\"):\n",
    "            _word_embeddings = tf.Variable(\n",
    "                self.embeddings,\n",
    "                dtype = tf.float32,\n",
    "                trainable = self.update_embedding,\n",
    "                name = \"_word_embeddings\")\n",
    "            \n",
    "            word_embeddings = tf.nn.embedding_lookup(\n",
    "                params = _word_embeddings,\n",
    "                ids = self.word_ids,\n",
    "                name = \"word_embeddings\")\n",
    "        self.word_embeddings = tf.nn.dropout(word_embeddings, self.dropout_pl)\n",
    "    def biLSTM_layer_op(self):\n",
    "        with tf.variable_scope(\"bi-lstm\"):\n",
    "            cell_fw = LSTMCell(self.hidden_dim)\n",
    "            cell_bw = LSTMCell(self.hidden_dim)\n",
    "            \n",
    "            (output_fw_seq , output_bw_seq) , _= tf.nn.bidirectional_dynamic_rnn(\n",
    "                cell_fw = cell_fw, \n",
    "                cell_bw = cell_bw,\n",
    "                inputs = word_embeddings,\n",
    "                sequence_lengths = self.sequence_lengths,\n",
    "                dtype = tf.float32\n",
    "            ) \n",
    "            \n",
    "            output = tf.concat([output_fw_seq, output_bw_seq], axis=-1)\n",
    "            output = tf.nn.dropout(output, self.dropout_pl)\n",
    "        with tf.variable_scope(\"proj\"):\n",
    "            W = tf.get_variable(name='W',\n",
    "                shape=[2*self.hidden_dim, self.num_tags],\n",
    "                initializer = tf.contrib.layers.xavier_initializer(),\n",
    "                dtype = tf.float32\n",
    "                               )\n",
    "            b = tf.get_variable(name=\"b\",\n",
    "                shape=[self.num_tags],\n",
    "                initializer = tf.zeros_initializer(),\n",
    "                dtype = tf.float32\n",
    "                               )\n",
    "            s = tf.shape(output)\n",
    "            \n",
    "            output = tf.reshape(output,[-1, 2*self.hidden_dim])\n",
    "            pred = tf.matmul(output, w)+b\n",
    "            \n",
    "            self.logits = tf.reshape(pred, [-1,s[1], self.num_tags])\n",
    "    def loss_op(self):\n",
    "        if self.CRF:\n",
    "            log_likelihood, self.transition_params = crf_log_likelihood(\n",
    "                inputs=self.logits,\n",
    "                tag_indices=self.labels,\n",
    "                sequence_lengths=self.sequence_lengths\n",
    "            )\n",
    "            self.loss = -tf.reduce_mean(log_likelihood)\n",
    "        else:\n",
    "            losses = tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "                logits = self.logits,\n",
    "                labels=self.labels\n",
    "            )\n",
    "            mask = tf.sequence_mask(self.sequence_lengths)\n",
    "            losses = tf.boolean_mask(losses, mask)\n",
    "            self.loss = tf.reduce_mean(losses)\n",
    "        tf.summary.scalar(\"loss\", self.loss)\n",
    "    \n",
    "    def softmax_pred_op(self):\n",
    "        if not self.CRF:\n",
    "            self.labels_softmax_ = tf.argmax(self.logits, axis=-1)\n",
    "            self.labels_softmax_ = tf.cast(self.labels_softmax_, tf.int32)\n",
    "    \n",
    "    def trainstep_op(self):\n",
    "        with tf.variable_scope(\"train_step\"):\n",
    "            self.global_step = tf.Variable(0, name=\"global_step\",trainable=False)\n",
    "            \n",
    "            if self.optimizer == 'Adam':\n",
    "                optim = tf.train.AdamOptimizer(learning_rate=self.lr_pl)\n",
    "            elif self.optimizer=='Adadelta':\n",
    "                optim = tf.train.AdadeltaOptimizer(learning_rate=self.lr_pl)\n",
    "            elif self.optimizer == 'Adagrad':\n",
    "                optim = tf.train.AdagradOptimizer(learning_rate=self.lr_pl)\n",
    "            elif self.optimizer=='RMSProp':\n",
    "                optim = tf.train.RMSPropOptimizer(learning_rate=self.lr_pl)\n",
    "            elif self.optimizer=='Momentum':\n",
    "                optim = tf.train.MomentumOptimizer(learning_rate=self.lr_pl, momentum=0.9)\n",
    "            elif self.optimizer=='SGD':\n",
    "                optim = tf.train.GradientDescentOptimizer(learning_rate=self.lr_pl)\n",
    "            else:\n",
    "                optim = tf.train.GradientDescentOptimizer(learning_rate=self.lr_pl)\n",
    "                \n",
    "            grads_and_vars = optim.compute_gradients(self.loss)\n",
    "            grads_and_vars_clip = [[tf.clip_by_value(g, -self.clip_grad, self.clip_grad), v] for g, v in grads_and_vars]    \n",
    "            self.train_op = optim.apply_gradients(grads_and_vars_clip,  global_step=self.global_step)\n",
    "            \n",
    "        def init_op(self):\n",
    "            self.init_op = tf.global_varibales_initializer()\n",
    "        \n",
    "        def add_summary(self, sess):\n",
    "            self.merged = tf.summary.merget_all()\n",
    "            self.file_writer = tf.Summary.FileWriter(self.summary_path, sess.graph)\n",
    "        \n",
    "        def train(self, train, dev):\n",
    "            saver = tf.train.Saver(tf.global_variables())\n",
    "            \n",
    "            with tf.Session() as sess:\n",
    "                sess.run(self.init_op)\n",
    "                self.add_summary(sess)\n",
    "                \n",
    "                for epoch in range(self.epoch_num):\n",
    "                    self.run_one_epoch(sess, train, dev, self.tag2label, epoch, saver)\n",
    "        def test(self, test):\n",
    "            saver = tf.train.Saver()\n",
    "            with tf.Session() as sess:\n",
    "                self.logger.info(\"============test==========\")\n",
    "                saver.restor(sess, self.model_path)\n",
    "                \n",
    "                label_list, seq_len_list = self.dev_one_epoch(sess, test)\n",
    "                self.evaluate(label_list, seq_len_list, test)\n",
    "        \n",
    "        def demo_one(self, sess, sent):\n",
    "            lablel_list = []\n",
    "            for seqs, labels in batch_yield(sent, self.batch_size,self.vocab,self.tag2label, shuffle=False):\n",
    "                label_list_, _ = self.predict_one_batch(sess, seqs)\n",
    "                label_list.extendz(label_list_)\n",
    "                \n",
    "            label2tag = {}\n",
    "            for tag, label in self.tag2label.items():\n",
    "                label2tag[label] = tag if label!=0 else label\n",
    "            tag = [label2tag[label] for label in label_list[0]]\n",
    "            \n",
    "            return tag\n",
    "        \n",
    "        def run_one_epoch(self, sess, train, dev, tag2label,epoch, saver):\n",
    "            num_batches = (len(train)+self.batch_size-1)//self.batch_size\n",
    "            self.logger_info(\"train lenght={} number_batches\".format(len(train), num_batches))\n",
    "            \n",
    "            start_time = time.strftime(\"%Y-%m-%d %H:%M:%S\", time.localtime())\n",
    "            batches = batch_yield(train, self.batch_size, self.vocab, self.tag2label,shuffle=self.shuffle)\n",
    "\n",
    "            for step ,(seq, labels) in enumerate(batches):\n",
    "                sys.stdout.write(' processing: {} batch / {} batches.'.format(step + 1, num_batches) + '\\r')\n",
    "                step_num = epoch*num_batches + step +1\n",
    "                feed_dict, _ = self.get_feed_dict(seqs, labels, self.lr, self.dropout_keep_prob)\n",
    "                \n",
    "                _, loss_train, summary, step_num_=  sess.run([self.train_op, self.loss, self.merged, self.global_step],\n",
    "                                                          feed_dict=feed_dict)\n",
    "                \n",
    "                if step+1 ==1 or (step+1) %300 ==0 or step+1 ==num_batches:\n",
    "                    self.logger.info(\n",
    "                        \"{ } epoch {}, step{}, loss:{:.4}, global_step:{}\".format(\n",
    "                        start_time, epoch + 1, step + 1,loss_train, step_num\n",
    "                        )\n",
    "                    )\n",
    "                self.file_writer.add_summary(summary, step_num)\n",
    "                \n",
    "                if step+1 == num_batches:\n",
    "                    self.logger.info(\"========save session========\")\n",
    "                    saver.save(sess, self.model_path, global_step = step_num)\n",
    "            self.logger.info(\"=============validation==========\")\n",
    "            lablel_list_dev , seq_len_list_dev = self.dev_one_epoch(sess, dev)\n",
    "            self.evaluate(label_list_dev, seq_len_list_dev, dev, epoch)\n",
    "            \n",
    "        def get_feed_dict(self, seqs, labels = None, lr = None, dropout = None):\n",
    "            word_ids, seq_len_list = pad_sequences(seqs, pad_mark=0)\n",
    "            feed_dict = {self.word_ids:word_ids,\n",
    "                        self.sequence_lengths : seq_len_list}\n",
    "            if labels is not None:\n",
    "                labels_, _ = pad_sequences(labels, pad_mark=0)\n",
    "                feed_dict[self.labels] = labels_\n",
    "            if lr is not None:\n",
    "                feed_dict[self.labels] = labels_\n",
    "            if dropout is not None:\n",
    "                feed_dict[self.dropout_pl] = dropout\n",
    "            \n",
    "            return feed_dict, seq_len_list\n",
    "        \n",
    "        def dev_one_epoch(self, sess, dev):\n",
    "            \n",
    "            label_list , seq_len_list = [], []\n",
    "            for seqs, labels in batch_yield(dev, self.batch_size, self.vocab,self.tag2label, shuffle=False):\n",
    "                label_list_, seq_len_list_ = self.predict_one_batch(sess, seqs)\n",
    "                \n",
    "                label_list.extend(label_list_)\n",
    "                seq_len_list.extend(seq_len_list_)\n",
    "                \n",
    "                return label_list, seq_len_list\n",
    "        \n",
    "        def predict_one_batch(self, sess, seqs):\n",
    "            feed_dict, seq_len_list = self.get_feed_dict(seqs, dropout=1.0)\n",
    "            \n",
    "            if self.CRF:\n",
    "                logits, transition_params = sess.run(\n",
    "                    [self.logits, self.transition_params],\n",
    "                    feed_dict = feed_dict)\n",
    "                label_list = []\n",
    "                for logit, seq_len in zip(logits, seq_len_list):\n",
    "                    viterbi_seq, _ = viterbi_decode(logit[:seq_len], transition_params)\n",
    "                    label_list.append(viterbi_seq)\n",
    "                return label_list, seq_len_list\n",
    "            else:\n",
    "                label_list = sess.run(self.labels_softmax_, feed_dict=feed_dict)\n",
    "                return label_list, seq_len_list\n",
    "            \n",
    "            def evaluate(self, label_list, seq_len_list, data, epoch=None):\n",
    "                \n",
    "                label2tag = {}\n",
    "                for tag, label in self.tag2label.items():\n",
    "                    label2tag[label] = tag if label!=0 else label\n",
    "                \n",
    "                model_predict = []\n",
    "                for label_, (sent, tag) in zip(label_list, data):\n",
    "                    tag_ = [label2tag[label__] for label__ in label_]\n",
    "                    sent_res = []\n",
    "                    if len(label_)!=len(sent):\n",
    "                        print(\"len=\",len(sent), len(label_),len(tag_))\n",
    "                    for i in range(len(sent)):\n",
    "                        sent_res.append([sent[i], tag[i], tag_[i]])\n",
    "                    model_predict.append(sent_res)\n",
    "                epoch_num = str(epoch+1) if epoch!=None else 'test'\n",
    "                label_path = os.path.join(self.result_path, 'label_'+epoch_num)\n",
    "                metric_path = os.path.join(self.result_path, 'result_metric_' + epoch_num)\n",
    "                \n",
    "                for item in conlleval(model_predict, label_path, metric_path):\n",
    "                    self.logger.info(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}