question-answer/4.dynamic-memory-gru.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from copy import deepcopy\n",
    "from attention_gru import AttentionGRUCell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "epoch = 10\n",
    "batch_size = 64\n",
    "size_layer = 64\n",
    "dropout_rate = 0.1\n",
    "n_hops = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BaseDataLoader():\n",
    "    def __init__(self):\n",
    "        self.data = {\n",
    "            'size': None,\n",
    "            'val':{\n",
    "                'inputs': None,\n",
    "                'questions': None,\n",
    "                'answers': None,},\n",
    "            'len':{\n",
    "                'inputs_len': None,\n",
    "                'inputs_sent_len': None,\n",
    "                'questions_len': None,\n",
    "                'answers_len': None}\n",
    "        }\n",
    "        self.vocab = {\n",
    "            'size': None,\n",
    "            'word2idx': None,\n",
    "            'idx2word': None,\n",
    "        }\n",
    "        self.params = {\n",
    "            'vocab_size': None,\n",
    "            '<start>': None,\n",
    "            '<end>': None,\n",
    "            'max_input_len': None,\n",
    "            'max_sent_len': None,\n",
    "            'max_quest_len': None,\n",
    "            'max_answer_len': None,\n",
    "        }\n",
    "        \n",
    "class DataLoader(BaseDataLoader):\n",
    "    def __init__(self, path, is_training, vocab=None, params=None):\n",
    "        super().__init__()\n",
    "        data, lens = self.load_data(path)\n",
    "        if is_training:\n",
    "            self.build_vocab(data)\n",
    "        else:\n",
    "            self.demo = data\n",
    "            self.vocab = vocab\n",
    "            self.params = deepcopy(params)\n",
    "        self.is_training = is_training\n",
    "        self.padding(data, lens)\n",
    "\n",
    "\n",
    "    def load_data(self, path):\n",
    "        data, lens = bAbI_data_load(path)\n",
    "        self.data['size'] = len(data[0])\n",
    "        return data, lens\n",
    "\n",
    "\n",
    "    def build_vocab(self, data):\n",
    "        signals = ['<pad>', '<unk>', '<start>', '<end>']\n",
    "        inputs, questions, answers = data\n",
    "        i_words = [w for facts in inputs for fact in facts for w in fact if w != '<end>']\n",
    "        q_words = [w for question in questions for w in question]\n",
    "        a_words = [w for answer in answers for w in answer if w != '<end>']\n",
    "        words = list(set(i_words + q_words + a_words))\n",
    "        self.params['vocab_size'] = len(words) + 4\n",
    "        self.params['<start>'] = 2\n",
    "        self.params['<end>'] = 3\n",
    "        self.vocab['word2idx'] = {word: idx for idx, word in enumerate(signals + words)}\n",
    "        self.vocab['idx2word'] = {idx: word for word, idx in self.vocab['word2idx'].items()}\n",
    "        \n",
    "\n",
    "    def padding(self, data, lens):\n",
    "        inputs_len, inputs_sent_len, questions_len, answers_len = lens\n",
    "\n",
    "        self.params['max_input_len'] = max(inputs_len)\n",
    "        self.params['max_sent_len'] = max([fact_len for batch in inputs_sent_len for fact_len in batch])\n",
    "        self.params['max_quest_len'] = max(questions_len)\n",
    "        self.params['max_answer_len'] = max(answers_len)\n",
    "\n",
    "        self.data['len']['inputs_len'] = np.array(inputs_len)\n",
    "        for batch in inputs_sent_len:\n",
    "            batch += [0] * (self.params['max_input_len'] - len(batch))\n",
    "        self.data['len']['inputs_sent_len'] = np.array(inputs_sent_len)\n",
    "        self.data['len']['questions_len'] = np.array(questions_len)\n",
    "        self.data['len']['answers_len'] = np.array(answers_len)\n",
    "        \n",
    "        inputs, questions, answers = deepcopy(data)\n",
    "        for facts in inputs:\n",
    "            for sentence in facts:\n",
    "                for i in range(len(sentence)):\n",
    "                    sentence[i] = self.vocab['word2idx'].get(sentence[i], self.vocab['word2idx']['<unk>'])\n",
    "                sentence += [0] * (self.params['max_sent_len'] - len(sentence))\n",
    "            paddings = [0] * self.params['max_sent_len']\n",
    "            facts += [paddings] * (self.params['max_input_len'] - len(facts))\n",
    "        for question in questions:\n",
    "            for i in range(len(question)):\n",
    "                question[i] = self.vocab['word2idx'].get(question[i], self.vocab['word2idx']['<unk>'])\n",
    "            question += [0] * (self.params['max_quest_len'] - len(question))\n",
    "        for answer in answers:\n",
    "            for i in range(len(answer)):\n",
    "                answer[i] = self.vocab['word2idx'].get(answer[i], self.vocab['word2idx']['<unk>'])\n",
    "\n",
    "        self.data['val']['inputs'] = np.array(inputs)\n",
    "        self.data['val']['questions'] = np.array(questions)\n",
    "        self.data['val']['answers'] = np.array(answers)\n",
    "        \n",
    "def bAbI_data_load(path, END=['<end>']):\n",
    "    inputs = []\n",
    "    questions = []\n",
    "    answers = []\n",
    "\n",
    "    inputs_len = []\n",
    "    inputs_sent_len = []\n",
    "    questions_len = []\n",
    "    answers_len = []\n",
    "\n",
    "    for d in open(path):\n",
    "        index = d.split(' ')[0]\n",
    "        if index == '1':\n",
    "            fact = []\n",
    "        if '?' in d:\n",
    "            temp = d.split('\\t')\n",
    "            q = temp[0].strip().replace('?', '').split(' ')[1:] + ['?']\n",
    "            a = temp[1].split() + END\n",
    "            fact_copied = deepcopy(fact)\n",
    "            inputs.append(fact_copied)\n",
    "            questions.append(q)\n",
    "            answers.append(a)\n",
    "\n",
    "            inputs_len.append(len(fact_copied))\n",
    "            inputs_sent_len.append([len(s) for s in fact_copied])\n",
    "            questions_len.append(len(q))\n",
    "            answers_len.append(len(a))\n",
    "        else:\n",
    "            tokens = d.replace('.', '').replace('\\n', '').split(' ')[1:] + END\n",
    "            fact.append(tokens)\n",
    "    return [inputs, questions, answers], [inputs_len, inputs_sent_len, questions_len, answers_len]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = DataLoader(path='qa5_three-arg-relations_train.txt',is_training=True)\n",
    "test_data = DataLoader(path='qa5_three-arg-relations_test.txt',is_training=False,\n",
    "                       vocab=train_data.vocab, params=train_data.params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "START = train_data.params['<start>']\n",
    "END = train_data.params['<end>']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shift_right(x):\n",
    "    batch_size = tf.shape(x)[0]\n",
    "    start = tf.to_int32(tf.fill([batch_size, 1], START))\n",
    "    return tf.concat([start, x[:, :-1]], 1)\n",
    "\n",
    "def GRU(name, rnn_size=None):\n",
    "    return tf.nn.rnn_cell.GRUCell(\n",
    "        rnn_size, kernel_initializer=tf.orthogonal_initializer(), name=name)\n",
    "\n",
    "def position_encoding(sentence_size, embedding_size):\n",
    "    encoding = np.ones((embedding_size, sentence_size), dtype=np.float32)\n",
    "    ls = sentence_size + 1\n",
    "    le = embedding_size + 1\n",
    "    for i in range(1, le):\n",
    "        for j in range(1, ls):\n",
    "            encoding[i-1, j-1] = (i - (le-1)/2) * (j - (ls-1)/2)\n",
    "    encoding = 1 + 4 * encoding / embedding_size / sentence_size\n",
    "    return np.transpose(encoding)\n",
    "\n",
    "def gen_episode(inputs_len, memory, q_vec, fact_vecs, proj_1, proj_2, attn_gru, is_training):\n",
    "    def gen_attn(fact_vec):\n",
    "        features = [fact_vec * q_vec,\n",
    "                    fact_vec * memory,\n",
    "                    tf.abs(fact_vec - q_vec),\n",
    "                    tf.abs(fact_vec - memory)]\n",
    "        feature_vec = tf.concat(features, 1)\n",
    "        attention = proj_1(feature_vec)\n",
    "        attention = proj_2(attention)\n",
    "        return tf.squeeze(attention, 1)\n",
    "\n",
    "    attns = tf.map_fn(gen_attn, tf.transpose(fact_vecs, [1,0,2]))\n",
    "    attns = tf.transpose(attns)                                      \n",
    "    attns = tf.nn.softmax(attns)                                     \n",
    "    attns = tf.expand_dims(attns, -1)                                \n",
    "    \n",
    "    _, episode = tf.nn.dynamic_rnn(attn_gru,\n",
    "                                   tf.concat([fact_vecs, attns], 2),\n",
    "                                   inputs_len,\n",
    "                                   dtype=np.float32)\n",
    "    return episode                  \n",
    "\n",
    "class QA:\n",
    "    def __init__(self, vocab_size):\n",
    "        self.questions = tf.placeholder(tf.int32,[None,None])\n",
    "        self.inputs = tf.placeholder(tf.int32,[None,None,None])\n",
    "        self.questions_len = tf.placeholder(tf.int32,[None])\n",
    "        self.inputs_len = tf.placeholder(tf.int32,[None])\n",
    "        self.answers_len = tf.placeholder(tf.int32,[None])\n",
    "        self.answers = tf.placeholder(tf.int32,[None,None])\n",
    "        self.training = tf.placeholder(tf.bool)\n",
    "        max_sent_len = train_data.params['max_sent_len']\n",
    "        max_quest_len = train_data.params['max_quest_len']\n",
    "        max_answer_len = train_data.params['max_answer_len']\n",
    "        \n",
    "        lookup_table = tf.get_variable('lookup_table', [vocab_size, size_layer], tf.float32)\n",
    "        lookup_table = tf.concat((tf.zeros([1, size_layer]), lookup_table[1:, :]), axis=0)\n",
    "        \n",
    "        cell_fw = GRU('cell_fw', size_layer // 2)\n",
    "        cell_bw = GRU('cell_bw', size_layer // 2)\n",
    "        inputs = tf.nn.embedding_lookup(lookup_table, self.inputs)\n",
    "        position = position_encoding(max_sent_len, size_layer)\n",
    "        inputs = tf.reduce_sum(inputs * position, 2)\n",
    "        birnn_out, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw,\n",
    "                                                   cell_bw,\n",
    "                                                   inputs,\n",
    "                                                   self.inputs_len,\n",
    "                                                   dtype=np.float32)\n",
    "        fact_vecs = tf.concat(birnn_out, -1)\n",
    "        fact_vecs = tf.layers.dropout(fact_vecs, dropout_rate, training=self.training)\n",
    "        \n",
    "        cell = GRU('question_rnn', size_layer)\n",
    "        questions = tf.nn.embedding_lookup(lookup_table, self.questions)\n",
    "        _, q_vec = tf.nn.dynamic_rnn(cell,\n",
    "                                 questions,\n",
    "                                 self.questions_len,\n",
    "                                 dtype=np.float32)\n",
    "        \n",
    "        proj_1 = tf.layers.Dense(size_layer, tf.tanh, name='attn_proj_1')\n",
    "        proj_2 = tf.layers.Dense(1, name='attn_proj_2')\n",
    "        attn_gru = AttentionGRUCell(size_layer, name='attn_gru')\n",
    "        memory_proj = tf.layers.Dense(size_layer, tf.nn.relu, name='memory_proj')\n",
    "        \n",
    "        memory = q_vec\n",
    "        for i in range(n_hops):\n",
    "            episode = gen_episode(self.inputs_len,\n",
    "                              memory,\n",
    "                              q_vec,\n",
    "                              fact_vecs,\n",
    "                              proj_1,\n",
    "                              proj_2,\n",
    "                              attn_gru,\n",
    "                              self.training)\n",
    "            memory = memory_proj(tf.concat([memory, episode, q_vec], 1))\n",
    "        \n",
    "        state_proj = tf.layers.Dense(size_layer, name='state_proj')\n",
    "        vocab_proj = tf.layers.Dense(vocab_size, name='vocab_proj')\n",
    "        \n",
    "        memory = tf.layers.dropout(memory, dropout_rate, training=self.training)\n",
    "        init_state = state_proj(tf.concat((memory, q_vec), -1))\n",
    "        helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "            inputs = tf.nn.embedding_lookup(lookup_table, shift_right(self.answers)),\n",
    "            sequence_length = self.answers_len)\n",
    "        decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "            cell = cell,\n",
    "            helper = helper,\n",
    "            initial_state = init_state,\n",
    "            output_layer = vocab_proj)\n",
    "        decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "            decoder = decoder)\n",
    "        self.training_id = decoder_output.sample_id\n",
    "        self.training_logits = decoder_output.rnn_output\n",
    "        \n",
    "        helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(\n",
    "            embedding = lookup_table,\n",
    "            start_tokens = tf.tile(\n",
    "                tf.constant([START], dtype=tf.int32), [tf.shape(init_state)[0]]),\n",
    "            end_token = END)\n",
    "        decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "            cell = cell,\n",
    "            helper = helper,\n",
    "            initial_state = init_state,\n",
    "            output_layer = vocab_proj)\n",
    "        decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "            decoder = decoder,\n",
    "            maximum_iterations = max_answer_len)\n",
    "        self.predict_id = decoder_output.sample_id\n",
    "        \n",
    "        self.cost = tf.reduce_mean(tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                        targets = self.answers,\n",
    "                                                        weights = tf.ones_like(self.answers, tf.float32)))\n",
    "        self.optimizer = tf.train.AdamOptimizer().minimize(self.cost)\n",
    "        \n",
    "        correct_pred = tf.equal(self.training_id, self.answers)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = QA(train_data.params['vocab_size'])\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, avg cost 1.193251, avg acc 0.595400\n",
      "epoch 2, avg cost 0.605435, avg acc 0.702650\n",
      "epoch 3, avg cost 0.309468, avg acc 0.866150\n",
      "epoch 4, avg cost 0.183491, avg acc 0.916000\n",
      "epoch 5, avg cost 0.163441, avg acc 0.917900\n",
      "epoch 6, avg cost 0.141715, avg acc 0.933100\n",
      "epoch 7, avg cost 0.103843, avg acc 0.963800\n",
      "epoch 8, avg cost 0.066167, avg acc 0.986200\n",
      "epoch 9, avg cost 0.051985, avg acc 0.993900\n",
      "epoch 10, avg cost 0.037008, avg acc 0.998250\n"
     ]
    }
   ],
   "source": [
    "for i in range(epoch):\n",
    "    total_cost, total_acc = 0, 0\n",
    "    for k in range(0, train_data.data['val']['inputs'].shape[0], batch_size):\n",
    "        index = min(k + batch_size, train_data.data['val']['inputs'].shape[0])\n",
    "        batch_questions = train_data.data['val']['questions'][k:index]\n",
    "        batch_inputs = train_data.data['val']['inputs'][k:index]\n",
    "        batch_inputs_len = train_data.data['len']['inputs_len'][k:index]\n",
    "        batch_questions_len = train_data.data['len']['questions_len'][k:index]\n",
    "        batch_answers_len = train_data.data['len']['answers_len'][k:index]\n",
    "        batch_answers = train_data.data['val']['answers'][k:index]\n",
    "        acc, cost, _ = sess.run([model.accuracy,model.cost,model.optimizer],\n",
    "                                  feed_dict={model.questions:batch_questions,\n",
    "                                            model.inputs:batch_inputs,\n",
    "                                            model.inputs_len:batch_inputs_len,\n",
    "                                            model.questions_len:batch_questions_len,\n",
    "                                            model.answers_len:batch_answers_len,\n",
    "                                            model.answers:batch_answers,\n",
    "                                            model.training:True})\n",
    "        total_cost += cost\n",
    "        total_acc += acc\n",
    "    total_cost /= (train_data.data['val']['inputs'].shape[0] / batch_size)\n",
    "    total_acc /= (train_data.data['val']['inputs'].shape[0] / batch_size)\n",
    "    print('epoch %d, avg cost %f, avg acc %f'%(i+1,total_cost,total_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "testing_size = 32\n",
    "batch_questions = test_data.data['val']['questions'][:testing_size]\n",
    "batch_inputs = test_data.data['val']['inputs'][:testing_size]\n",
    "batch_inputs_len = test_data.data['len']['inputs_len'][:testing_size]\n",
    "batch_questions_len = test_data.data['len']['questions_len'][:testing_size]\n",
    "batch_answers_len = test_data.data['len']['answers_len'][:testing_size]\n",
    "batch_answers = test_data.data['val']['answers'][:testing_size]\n",
    "logits = sess.run(model.predict_id,\n",
    "                        feed_dict={model.questions:batch_questions,\n",
    "                                   model.inputs:batch_inputs,\n",
    "                                   model.inputs_len:batch_inputs_len,\n",
    "                                   model.questions_len:batch_questions_len,\n",
    "                                   model.answers_len:batch_answers_len,\n",
    "                                   model.training:False})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "QUESTION: What did Fred give to Jeff ? <pad>\n",
      "REAL: football\n",
      "PREDICT: football \n",
      "\n",
      "QUESTION: Who gave the football to Jeff ? <pad>\n",
      "REAL: Fred\n",
      "PREDICT: Fred \n",
      "\n",
      "QUESTION: What did Fred give to Jeff ? <pad>\n",
      "REAL: football\n",
      "PREDICT: football \n",
      "\n",
      "QUESTION: Who did Fred give the football to ?\n",
      "REAL: Jeff\n",
      "PREDICT: Jeff \n",
      "\n",
      "QUESTION: Who did Jeff give the football to ?\n",
      "REAL: Fred\n",
      "PREDICT: Fred \n",
      "\n",
      "QUESTION: Who gave the apple ? <pad> <pad> <pad>\n",
      "REAL: Jeff\n",
      "PREDICT: Jeff \n",
      "\n",
      "QUESTION: Who received the apple ? <pad> <pad> <pad>\n",
      "REAL: Bill\n",
      "PREDICT: Bill \n",
      "\n",
      "QUESTION: What did Bill give to Fred ? <pad>\n",
      "REAL: apple\n",
      "PREDICT: apple \n",
      "\n",
      "QUESTION: What did Bill give to Fred ? <pad>\n",
      "REAL: apple\n",
      "PREDICT: apple \n",
      "\n",
      "QUESTION: Who received the apple ? <pad> <pad> <pad>\n",
      "REAL: Fred\n",
      "PREDICT: Fred \n",
      "\n",
      "QUESTION: Who received the milk ? <pad> <pad> <pad>\n",
      "REAL: Mary\n",
      "PREDICT: Mary \n",
      "\n",
      "QUESTION: What did Mary give to Bill ? <pad>\n",
      "REAL: milk\n",
      "PREDICT: milk \n",
      "\n",
      "QUESTION: Who received the milk ? <pad> <pad> <pad>\n",
      "REAL: Bill\n",
      "PREDICT: Bill \n",
      "\n",
      "QUESTION: Who did Mary give the milk to ?\n",
      "REAL: Bill\n",
      "PREDICT: Bill \n",
      "\n",
      "QUESTION: What did Mary give to Bill ? <pad>\n",
      "REAL: milk\n",
      "PREDICT: milk \n",
      "\n",
      "QUESTION: Who received the football ? <pad> <pad> <pad>\n",
      "REAL: Jeff\n",
      "PREDICT: Jeff \n",
      "\n",
      "QUESTION: Who gave the football to Jeff ? <pad>\n",
      "REAL: Fred\n",
      "PREDICT: Fred \n",
      "\n",
      "QUESTION: What did Fred give to Jeff ? <pad>\n",
      "REAL: football\n",
      "PREDICT: football \n",
      "\n",
      "QUESTION: Who received the football ? <pad> <pad> <pad>\n",
      "REAL: Jeff\n",
      "PREDICT: Jeff \n",
      "\n",
      "QUESTION: Who gave the milk ? <pad> <pad> <pad>\n",
      "REAL: Mary\n",
      "PREDICT: Mary \n",
      "\n",
      "QUESTION: What did Jeff give to Mary ? <pad>\n",
      "REAL: milk\n",
      "PREDICT: milk \n",
      "\n",
      "QUESTION: Who gave the milk to Jeff ? <pad>\n",
      "REAL: Mary\n",
      "PREDICT: Mary \n",
      "\n",
      "QUESTION: Who gave the milk to Jeff ? <pad>\n",
      "REAL: Mary\n",
      "PREDICT: Mary \n",
      "\n",
      "QUESTION: Who gave the apple to Jeff ? <pad>\n",
      "REAL: Bill\n",
      "PREDICT: Bill \n",
      "\n",
      "QUESTION: Who gave the apple ? <pad> <pad> <pad>\n",
      "REAL: Jeff\n",
      "PREDICT: Jeff \n",
      "\n",
      "QUESTION: What did Jeff give to Fred ? <pad>\n",
      "REAL: apple\n",
      "PREDICT: apple \n",
      "\n",
      "QUESTION: Who received the apple ? <pad> <pad> <pad>\n",
      "REAL: Fred\n",
      "PREDICT: Fred \n",
      "\n",
      "QUESTION: Who received the apple ? <pad> <pad> <pad>\n",
      "REAL: Jeff\n",
      "PREDICT: Jeff \n",
      "\n",
      "QUESTION: Who gave the apple ? <pad> <pad> <pad>\n",
      "REAL: Jeff\n",
      "PREDICT: Jeff \n",
      "\n",
      "QUESTION: Who did Jeff give the apple to ?\n",
      "REAL: Fred\n",
      "PREDICT: Fred \n",
      "\n",
      "QUESTION: Who gave the milk ? <pad> <pad> <pad>\n",
      "REAL: Mary\n",
      "PREDICT: Mary \n",
      "\n",
      "QUESTION: Who gave the milk ? <pad> <pad> <pad>\n",
      "REAL: Bill\n",
      "PREDICT: Bill \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(testing_size):\n",
    "    print('QUESTION:',' '.join([train_data.vocab['idx2word'][k] for k in batch_questions[i]]))\n",
    "    print('REAL:',train_data.vocab['idx2word'][batch_answers[i,0]])\n",
    "    print('PREDICT:',train_data.vocab['idx2word'][logits[i,0]],'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}