split_text_data_proc_and_train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "level = logging.getLevelName(\"INFO\")\n",
    "logging.basicConfig(\n",
    "  level=level,\n",
    "  format=\"[%(asctime)s %(levelname)s] %(message)s\",\n",
    "  datefmt=\"%H:%M:%S\",\n",
    ")\n",
    "import json\n",
    "import numpy as np\n",
    "import parallel\n",
    "import torch\n",
    "\n",
    "from experiment import Net, init_dataset, run_one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "69"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chars = [' ', '!', '\"', '#', '$', '%', '&', \"'\", '(', ')', '*', '+', ',', '-', '.', '/',\n",
    "         '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',\n",
    "         ':', ';', '<', '=', '>', '?', '@', '[', '\\\\', ']', '^', '_', '`', \n",
    "         'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p',\n",
    "         'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', \n",
    "         '{', '|', '}', '~']\n",
    "len(chars)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "70"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_to_idx_map = {char: chars.index(char)+1 for char in chars}\n",
    "# add chr 0 as an unknown char id\n",
    "char_to_idx_map[chr(0)] = 0\n",
    "idx_to_char_map = {val: key for key, val in char_to_idx_map.items()}\n",
    "len(char_to_idx_map) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def RecoverCharacters(encoded):\n",
    "  chars = []\n",
    "  for ci in encoded:\n",
    "    chars.append(idx_to_char_map[ci])\n",
    "  return chars\n",
    "def RecoverOriginalSequence(encoded, spaces):\n",
    "  s = \"\"\n",
    "  for i, c in enumerate(RecoverCharacters(encoded)):\n",
    "    s+=c\n",
    "    if spaces[i] >= 1:\n",
    "      s+=\" \"\n",
    "  return s\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def GetEncodingAndSpaceLabels(chunk):\n",
    "  # get original chunk, integer encoded char with spaces removed, and space location vector\n",
    "  space_location = [0]*CHUNK_SIZE\n",
    "  spaces = 0\n",
    "  encoded_chars = []\n",
    "  for i, c in enumerate(chunk):\n",
    "    if c == \" \":\n",
    "      space_location[i - 1 - spaces] = 1\n",
    "      spaces += 1\n",
    "    else:\n",
    "      encoded_chars.append(char_to_idx_map.get(c, 0))\n",
    "  return chunk, encoded_chars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def GetCharEncoding(text):\n",
    "  encoding = []\n",
    "  for c in text:\n",
    "    encoding.append( char_to_idx_map.get(c, 0))\n",
    "  return torch.tensor(encoding, dtype=torch.long).unsqueeze(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# Let me use the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "h = {\n",
    "  'kernel|filter_sizes': [\n",
    "    (4,32),\n",
    "    (4,64),\n",
    "    (4,64),\n",
    "  ],\n",
    "  'final_conv_kernel': 3,  \n",
    "  'sequence_length': 100,\n",
    "  'vocab_size': 70, #character vocab\n",
    "  'char_embedding_size': 8,\n",
    "  'conv_activation': 'relu',\n",
    "  }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = Net(h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(torch.load('model_trained_on_wiki_data.pt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def SplitText(text, model):\n",
    "  inp_tensor = GetCharEncoding(text)\n",
    "  out = model(inp_tensor)\n",
    "  space_location = (out>.5).int()\n",
    "  return RecoverOriginalSequence(inp_tensor[0].cpu().numpy(), space_location[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'pets mart'"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SplitText(\"petsmart\", model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'this string is justatest'"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"thisstringisjustatest\"\n",
    "SplitText(text, model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# Data preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# raw text file\n",
    "text_file = 'yourdata.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lines = []\n",
    "with open(text_file) as f:\n",
    "  for l in f.readlines():\n",
    "    line = l.lower().strip()\n",
    "    line = \"\".join([x if x in char_to_idx_map else idx_to_char_map[0] for x in line]).strip()\n",
    "    lines.append(line)\n",
    "\n",
    "full_set = \" \".join(lines)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "548247"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# How many characters in a chunk of text after spaces are removed\n",
    "CHUNK_SIZE = 100\n",
    "chunks = []\n",
    "count = 0\n",
    "cur_chunk = []\n",
    "for c in full_set:\n",
    "  if c == \" \":\n",
    "    if count != 0:\n",
    "      cur_chunk.append(c)\n",
    "    continue\n",
    "  if count >= CHUNK_SIZE:\n",
    "    chunks.append(\"\".join(cur_chunk))\n",
    "    cur_chunk = []\n",
    "    count = 0\n",
    "  \n",
    "  if count == 0 and c==\" \":\n",
    "   continue\n",
    "  count +=1\n",
    "  cur_chunk.append(c)\n",
    "\n",
    "len(chunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "original_chunk, char_ids, spaces = GetEncodingAndSpaceLabels(\"convert this to a chunk with no spaces and get a vector indicating where spaces belong\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0,\n",
       "       1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(spaces)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "out = parallel.RunInParallel(chunks, GetEncodingAndSpaceLabels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.random.shuffle(out)\n",
    "\n",
    "chunks, encoded, spaces = zip(*out)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "i = 7\n",
    "assert chunks[i] == RecoverOriginalSequence(encoded[i], spaces[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "raw_data = {'char_as_ints': encoded, 'space_labels': spaces, 'char_to_idx_map': char_to_idx_map}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "torch.save({'features': torch.tensor(encoded), 'labels': torch.tensor(labels), 'char_to_idx_map':data['char_to_idx_map']}, 'data/removed_spaces.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Training the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "h = {\n",
    "  'dataset':'torch_wiki.pt',\n",
    "  'dataset_split': [.5,.1],\n",
    "  'batch_size': 128,\n",
    "  'learning_rate': .5,\n",
    "  'momentum': .98,\n",
    "  'epochs': 3,\n",
    "  'kernel|filter_sizes': [\n",
    "    (4,32),\n",
    "    (4,64),\n",
    "    (4,64),\n",
    "  ],\n",
    "  'final_conv_kernel': 3,  \n",
    "  'sequence_length': 100,\n",
    "  'vocab_size': 70, #character vocab\n",
    "  'char_embedding_size': 8,\n",
    "  'conv_activation': 'relu',\n",
    "  'lr_step_size': 4000,\n",
    "  'lr_decay': .9,\n",
    "  'run_validation':True,\n",
    "  }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ds = init_dataset(h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'d from the position . mike steele then took the job from banister in 2009 . banister managed the scottsdale scorpions of t'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d = ds[0][0]\n",
    "RecoverOriginalSequence(d['features'].cpu().numpy(), d['labels'].cpu().numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:40:31 INFO] Experiment Model:\n",
      "Net(\n",
      "  (emb): Embedding(70, 8)\n",
      "  (convs): Sequential(\n",
      "    (0): Conv1d(8, 32, kernel_size=(4,), stride=(1,), padding=(2,))\n",
      "    (1): LambdaLayer()\n",
      "    (2): ReLU()\n",
      "    (3): Conv1d(32, 64, kernel_size=(4,), stride=(1,), padding=(2,))\n",
      "    (4): LambdaLayer()\n",
      "    (5): ReLU()\n",
      "    (6): Conv1d(64, 64, kernel_size=(4,), stride=(1,), padding=(2,))\n",
      "    (7): LambdaLayer()\n",
      "    (8): ReLU()\n",
      "  )\n",
      "  (conv_final): Conv1d(64, 1, kernel_size=(3,), stride=(1,), padding=(1,))\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:40:31 INFO] Hyperparams:\n",
      "{'dataset': 'torch_wiki.pt', 'dataset_split': [0.5, 0.1], 'batch_size': 128, 'learning_rate': 0.5, 'momentum': 0.98, 'epochs': 3, 'kernel|filter_sizes': [(4, 32), (4, 64), (4, 64)], 'final_conv_kernel': 3, 'sequence_length': 100, 'vocab_size': 70, 'char_embedding_size': 8, 'conv_activation': 'relu', 'lr_step_size': 400, 'lr_decay': 0.9, 'run_validation': True}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:40:31 INFO] Model Size: 1.06e+05 bytes\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:40:31 INFO] TRAINING\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:40:31 INFO] total batches: 50217\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:40:39 INFO] Log Type  global_step  epoch  char_accuracy  example_accuracy  loss     lr_at_step  us/ex  % cmplt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:40:39 INFO] TRAIN     502          0      0.95554685592  0.0390625         0.11003  0.45        124.3  0.999  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:40:47 INFO] TRAIN     1004         0      0.96531248092  0.0859375         0.09827  0.405       125.8  1.999  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:40:55 INFO] TRAIN     1506         0      0.97101563215  0.078125          0.08024  0.36450000  124.4  2.998  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:41:04 INFO] TRAIN     2008         0      0.96999996900  0.1015625         0.08492  0.29524500  127.0  3.998  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:41:11 INFO] TRAIN     2510         0      0.97304683923  0.1015625         0.07336  0.26572050  125.6  4.998  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:41:21 INFO] TRAIN     3012         0      0.97429686784  0.125             0.06764  0.23914845  130.6  5.997  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:41:30 INFO] TRAIN     3514         0      0.97421872615  0.1484375         0.06845  0.21523360  130.3  6.997  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:41:38 INFO] TRAIN     4016         0      0.97226560115  0.109375          0.07200  0.17433922  130.1  7.997  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:41:46 INFO] TRAIN     4518         0      0.97499996423  0.1015625         0.06779  0.15690529  129.4  8.996  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:41:55 INFO] TRAIN     5020         0      0.97609370946  0.1640625         0.06901  0.14121476  130.5  9.996  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:41:55 INFO] TRAIN     5021         0      0.97585934400  0.1796875         0.06270  0.14121476  130.5  9.998  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:43:21 INFO] TRAIN     10042        0      0.97679686546  0.1640625         0.06655  0.03589489  131.9  19.99  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:44:52 INFO] TRAIN     15063        0      0.97882813215  0.1953125         0.06239  0.01013777  135.3  29.99  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:46:12 INFO] TRAIN     20084        1      0.97820311784  0.2265625         0.05855  0.00257688  124.6  39.99  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:47:36 INFO] TRAIN     25105        1      0.97609370946  0.15625           0.06150  0.00072778  128.5  49.99  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:49:02 INFO] TRAIN     30126        1      0.97921872138  0.125             0.05970  0.00018499  130.8  59.99  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:50:23 INFO] TRAIN     35147        2      0.97679686546  0.1484375         0.06205  5.22478381  120.3  69.99  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:51:40 INFO] TRAIN     40168        2      0.9765625      0.15625           0.06417  1.32806994  119.2  79.98  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:52:51 INFO] TRAIN     45189        2      0.97945308685  0.234375          0.05702  3.75086178  115.7  89.98  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:54:03 INFO] TRAIN     50210        2      0.97789061069  0.1171875         0.05787  9.53418740  114.3  99.98  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[01:54:16 INFO] VAL       50220               0.97768712043  0.17460158467292  0.06188                            \n"
     ]
    }
   ],
   "source": [
    "results, model = run_one(h, ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "text = \"petsmart\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'pets mart'"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SplitText(text, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top Splits:\n",
      "pets mart\n",
      "petsmart\n",
      "petsmart \n",
      "petsmart\n",
      "petsmart\n",
      "pet smart\n"
     ]
    }
   ],
   "source": [
    "inp_tensor = GetCharEncoding(text)\n",
    "out = model(inp_tensor)\n",
    "space_location = (out>.5).int()\n",
    "\n",
    "print(\"Top Splits:\")\n",
    "for location in out.argsort(descending=True)[0][:6]:\n",
    "  space_location = torch.zeros(size=space_location.shape)\n",
    "  space_location[0][location] = 1\n",
    "  print(RecoverOriginalSequence(inp_tensor[0].cpu().numpy(), space_location[0]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "argv": [
    "/usr/bin/python3",
    "-m",
    "ipykernel_launcher",
    "-f",
    "{connection_file}"
   ],
   "display_name": "Python 3",
   "env": null,
   "interrupt_mode": "signal",
   "language": "python",
   "metadata": null,
   "name": "python3"
  },
  "name": "split_text_data_proc_and_train.ipynb"
 },
 "nbformat": 4,
 "nbformat_minor": 2
}