composeRNN1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pygame.midi\n",
    "import time\n",
    "\n",
    "pygame.midi.init()\n",
    "player = pygame.midi.Output(0)\n",
    "player.set_instrument(48) # https://pjb.com.au/muscript/gm.html\n",
    "player.note_on(64, 127) # note_on(note, velocity=None, channel = 0) https://en.scratch-wiki.info/wiki/MIDI_Notes\n",
    "time.sleep(2)\n",
    "player.note_off(64, 127)\n",
    "del player\n",
    "pygame.midi.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data has 36979 characters, 14 unique.\n",
      "----\n",
      " 232685'7'8,98:2,5:27 '083:74798293823:174:970:9'8,21 8806:423484200233670,14'7'2'137138445,3:8:199,1:9,29669:248988'0'5'6'5:8,32772:71:74103063 2939047 52 '9757616 1,7 6'29 568,8'21,::8:95344 2182':,2141:,:9'9','768'0,077914' 5,'8017'2 43787798  5:57011263523,3290604,9945'''3'4''41 3352,88' 686406,7'73:32587:1832' ' 8 395677205128 5000619077'2306,8 3165:42:95365951 8'21462,,34 6',5 4:418 1 64,:6354, '0477985''''3:825,819672,2:644'83676,'' 509'5,',2'2028,60,02049245 80,12,76'7818 :'58'60076071310 \n",
      "----\n",
      "iter 0, loss: 65.976432\n",
      "----\n",
      " 0'''20', ''',50'0', 7', '':88', '1, ',7'1, 7' 0:, ',,0',:8',217 7'1:50', ',703:3 ', '' ':70', 180378: 0' :9:81', '1:0',:7:, ', 00, '0, '', ', 01071,'78:8', :54, 0::006 '1:5,92'9:87, 6'067'  '1072'0'6  ',:8 ,, 90 ',  ', 0,, '11 ,, '1:  , 0''  '1 ',, '1:0'0 '1: 0',0:20',773',50' '', '9:59: :5, ',, 1,81:008'  ',5700 2:7'', 0 0:8'10'::45171',0'1:70', '0 ','2,, '' 2'9,71: 8,5 5', '1:80'2'6,0'8,7''4:850'8:8,9271:0',,0'2'2:5 '':77,0',  '1:72', ', '', 0'2720'1100',2', :,'2:0':0',:'0'71:8::9 ',0028:68'67 \n",
      "----\n",
      "iter 100, loss: 65.782784\n",
      "----\n",
      " 70:80', '1:70:8:0', '0:65:80', '0:65:80', ', '0:70:67', '0:75:,8'1:75:0', '1:62:0', '0:70:0', '0:51:80:, '0:70:802'0'7 ', '''0:70', '0:70:82:0',1:800, '0:78:0', '0:54:80', '2:74:80', 57:68:0', '0:799, '0:72:6'0:75:0', '2:80:3', '0:45:0', '0366:0', ',:78:0', '2566:0', '0:70:7:70:0'607,0', '2:70:80':7780', '0:60:0', '0'77:80', ', ':80'80:82:40'2:61:0', ', '1:70:883', '0:55:0', '2::5:80', 0, '0:75:882787:5'5 0'  '2:50:80', '0:70:0', '', '80', '0:76', '0:75:0:0', '1: 0:70', '0:67:80', '2:48:0', '2:8 \n",
      "----\n",
      "iter 200, loss: 61.968914\n",
      "----\n",
      " 70:0', '2:48:80', '0:70:0', '1:62:80', '2:70:0', '0:70:80'0:75:80', '9:69:80', '2:65:80, '0:67:0', '1:52:80', '0:67:0:, '1:80::0', '2:55:0', ',:73:0', '2:89:0', '0:79:80', '1:79:80', '0:51:50', '0:71:0', '1:75:82:0', '1:75:0', '0:79:80', '0:72:80', '0:75:80', '1:82:80', '2:51:0', '0:79:80', '0:5:50', '0:64:2:0,, '0:74:0', '2:58', '2:73:60', '2:80:0', '1:69:0', '2:75:880:76:80', '0:75:80', '1:43: ', '0:79:0', ', '2:70:80'0:70:0',0'0:77:0', '1:75:0', '1:79:80', '1:78:0', '0:74:0', '1:72:0', '0:77: \n",
      "----\n",
      "iter 300, loss: 57.475198\n",
      "----\n",
      " :80', '0:75:880', '0:76:80', '1:70:0', '2:51:80', '1:71:80', '1:72:80', '1:82:0', '0:43:8 ', '0:72:0', '0:51:80', '2:40:0', '0:57:0', '2:71:80', '0:77:0', '2:65:80', '0:61:80', '0:78:0', '1:75:0', '1:74:0', '1:43:0', '1:76:70', '1:7 :0', '0:79:77', '1:72:0', '0:880', '1:47:0', '0:72:80', '0:79:80', '0:86:0', '0:57:80', '2:74:0', '0:81:80', '0:75:0', '0:81:0', '0:75:0', '0:75:0', '1:72:80'2 '0:82:0', '1:05 ', '1:83:0', '0:74:80', '1:51:80', '0:71:0: '0'70:80', '0:89:0', '0:65:80', '1:75:0', '1:54 \n",
      "----\n",
      "iter 400, loss: 53.235820\n",
      "----\n",
      " :80', '1:74:80'0 '2:88:0', '1:61:0', '1:70:80', '1:78:80', '1:51:80', '0:77:80', '0:47:80', '1:71:80', '0:72:0', '0:80:0', '1:42:80', '1:74:0', '1:5 :80', '4:52:0', '0:75:80', ',:40:80', '09:7:80', '1:54:0', '1:62:0', '0:75:80', '1:70:0', '0:44:0', '0:77:80', '2:58:0', '0:75:0', '0:70:0', '2:50:0', '2:40:0', '1:59:0', '0:70:00','49:80', '0:79:80', '1:70:0', '1:56:80', '1:44:0', '0:72:80', '1:72:0', '2:50:80', '2:70:80', '1:61:0', '0:75:80', '1:70:0', '0:70:0', '1:75:80', '1:40:80', '3:46:80', '0 \n",
      "----\n",
      "iter 500, loss: 49.290967\n",
      "----\n",
      " 0', '0:57:0', '0:71:60', '1:70:0', '1:70:80', '0:74:0', '1:52:70', '1:67:80', '2:48:0', '1:72:0', '0:7470', '2:55:0', '0:69:80', '2:72:80', '2:55:0', '2:53:80', '2:41:0', '0:79:80', '2:70:80', '1:69:80', '1:60:80', '2:46:80''2:59:0', '2:52:6', '0:69:0', '2:46:0', '1:65:80', '0:70:80', '2:54:80', '1:63:80', '1:69:0', '0:79:80', '1:74:80', '2:48:0', '2:73:80', '2:50:00, '2:50:0', '2:50', '0:64:80', '0::5:0', '2:51:80', '2:53:80', '2:53:80', '2:52:0', '2:53:0', '6:52:0', '2:67:0', '2:69:0', '0:79:0 \n",
      "----\n",
      "iter 600, loss: 45.810499\n",
      "----\n",
      " 0', '2:53:0', '0:46:80', '2:59:0', '2:51:0', '2:43:0', '0:64:80', '0:79:80', '2:40:0', '1:75:80', '2:41:80', '2:90:0', '0:75:80', '2:53:0', '2:58:80', '1:72:0', '0:58:80', '2:59:0', '0:63:80', '2:53:0', '2:57:80', '1:72:80', '2:6::0', '2:53:0', '2:43:80', '2:47:0', ',:58:0', '1:74:80', '2:50:0', '0:72:0', '0:70:80', '1:79:80', '2:53:0', '0:65:80', '1:74:0', '1:70:80', '2:53:0', '2:50:80', '1:77:6'',:'0:70:0', '2:53:80', '2:43:0', '2:53:0', '0:70:80'' '1:73:80', '2:53:0', '0:76:0', '0:79:80', '2: \n",
      "----\n",
      "iter 700, loss: 42.618097\n",
      "----\n",
      " , '1:84:0', '0:74:0', '1:80:0', '2:51:80', '2:59:0', '0:75:0', '0:75:80', '1:81:0', '2:43:0', '1:8 :80', '1:64:0', '0:72:80', '2:85:0', '0:75:0', '0:74:80', '0:77:0', '2:51:0', '0:72:80', '0:80:80', '1:72:0', '0:80:80', '0:75:80', '1:80:80', '1:55:0', '2:5::0', '2:92:0', '2:48:80', '2:51:80', '2:52:0', '::63:0', '1:84:80', ''7:8:80', ', '0:71:80', '1:72:80', '2:44:0', '1:60:0', '0:73:80', '0:64:80', '2:70:80', '0:75:0', '0:76:0', '2:50:80', '2:50:80', '2:48:80'' '2:48:0', '2:43:0', '2:46:80', '1 \n",
      "----\n",
      "iter 800, loss: 39.726151\n",
      "----\n",
      "  '1:69:0', '2:67:80', '0:62:80', '0:69:0', '0:70:80', '0:75:0', '1:77:0', ' :74:88'' '1:40:0', '0:80:80', '1:74:80', '1:79:0', '1:73:80', '1:72:80', '0:75:0', '1:63:0', '0:67:80', 720'8:0', '0:75:0', '1:70:0', '2:52:80', '2:65:0', '0:60:0', '1:70:80', '2:45:0', '2:52:0', '2:69:0', '0:77:0', '1:60:0', '2:60:0', '1:69:0', '0:79:80', '1:6580', '0:70:80', '0:75:0', '1:70:0', '1:54:0', '1:70:0', '2:70:0', '0:74:0', '0:69:0', '1:79:0', '2:50:0', '2:55:0', '0:74:80', '2:50:0', '1:67:0', '0:65:0', '2:60 \n",
      "----\n",
      "iter 900, loss: 37.081441\n"
     ]
    }
   ],
   "source": [
    "# from https://gist.github.com/karpathy/d4dee566867f8291f086\n",
    "\n",
    "\"\"\"\n",
    "Minimal character-level Vanilla RNN model. Written by Andrej Karpathy (@karpathy)\n",
    "BSD License\n",
    "\"\"\"\n",
    "import numpy as np\n",
    "\n",
    "nruns = 1000\n",
    "noutput = 100\n",
    "\n",
    "# data I/O\n",
    "data = open('notes0.txt', 'r').read() # should be simple plain text file\n",
    "chars = list(set(data))\n",
    "data_size, vocab_size = len(data), len(chars)\n",
    "print('data has %d characters, %d unique.' % (data_size, vocab_size))\n",
    "char_to_ix = { ch:i for i,ch in enumerate(chars) }\n",
    "ix_to_char = { i:ch for i,ch in enumerate(chars) }\n",
    "\n",
    "# hyperparameters\n",
    "hidden_size = 100 # size of hidden layer of neurons\n",
    "seq_length = 25 # number of steps to unroll the RNN for\n",
    "learning_rate = 1e-1\n",
    "\n",
    "# model parameters\n",
    "Wxh = np.random.randn(hidden_size, vocab_size)*0.01 # input to hidden\n",
    "Whh = np.random.randn(hidden_size, hidden_size)*0.01 # hidden to hidden\n",
    "Why = np.random.randn(vocab_size, hidden_size)*0.01 # hidden to output\n",
    "bh = np.zeros((hidden_size, 1)) # hidden bias\n",
    "by = np.zeros((vocab_size, 1)) # output bias\n",
    "\n",
    "def lossFun(inputs, targets, hprev):\n",
    "  \"\"\"\n",
    "  inputs,targets are both list of integers.\n",
    "  hprev is Hx1 array of initial hidden state\n",
    "  returns the loss, gradients on model parameters, and last hidden state\n",
    "  \"\"\"\n",
    "  xs, hs, ys, ps = {}, {}, {}, {}\n",
    "  hs[-1] = np.copy(hprev)\n",
    "  loss = 0\n",
    "  # forward pass\n",
    "  for t in range(len(inputs)):\n",
    "    xs[t] = np.zeros((vocab_size,1)) # encode in 1-of-k representation\n",
    "    xs[t][inputs[t]] = 1\n",
    "    hs[t] = np.tanh(np.dot(Wxh, xs[t]) + np.dot(Whh, hs[t-1]) + bh) # hidden state\n",
    "    ys[t] = np.dot(Why, hs[t]) + by # unnormalized log probabilities for next chars\n",
    "    ps[t] = np.exp(ys[t]) / np.sum(np.exp(ys[t])) # probabilities for next chars\n",
    "    loss += -np.log(ps[t][targets[t],0]) # softmax (cross-entropy loss)\n",
    "  # backward pass: compute gradients going backwards\n",
    "  dWxh, dWhh, dWhy = np.zeros_like(Wxh), np.zeros_like(Whh), np.zeros_like(Why)\n",
    "  dbh, dby = np.zeros_like(bh), np.zeros_like(by)\n",
    "  dhnext = np.zeros_like(hs[0])\n",
    "  for t in reversed(range(len(inputs))):\n",
    "    dy = np.copy(ps[t])\n",
    "    dy[targets[t]] -= 1 # backprop into y. see http://cs231n.github.io/neural-networks-case-study/#grad if confused here\n",
    "    dWhy += np.dot(dy, hs[t].T)\n",
    "    dby += dy\n",
    "    dh = np.dot(Why.T, dy) + dhnext # backprop into h\n",
    "    dhraw = (1 - hs[t] * hs[t]) * dh # backprop through tanh nonlinearity\n",
    "    dbh += dhraw\n",
    "    dWxh += np.dot(dhraw, xs[t].T)\n",
    "    dWhh += np.dot(dhraw, hs[t-1].T)\n",
    "    dhnext = np.dot(Whh.T, dhraw)\n",
    "  for dparam in [dWxh, dWhh, dWhy, dbh, dby]:\n",
    "    np.clip(dparam, -5, 5, out=dparam) # clip to mitigate exploding gradients\n",
    "    clip_diag(dparam, 1)\n",
    "  return loss, dWxh, dWhh, dWhy, dbh, dby, hs[len(inputs)-1]\n",
    "\n",
    "def sample(h, seed_ix, n):\n",
    "  \"\"\" \n",
    "  sample a sequence of integers from the model \n",
    "  h is memory state, seed_ix is seed letter for first time step\n",
    "  \"\"\"\n",
    "  x = np.zeros((vocab_size, 1))\n",
    "  x[seed_ix] = 1\n",
    "  ixes = []\n",
    "  for t in range(n):\n",
    "    h = np.tanh(np.dot(Wxh, x) + np.dot(Whh, h) + bh)\n",
    "    y = np.dot(Why, h) + by\n",
    "    p = np.exp(y) / np.sum(np.exp(y))\n",
    "    ix = np.random.choice(range(vocab_size), p=p.ravel())\n",
    "    x = np.zeros((vocab_size, 1))\n",
    "    x[ix] = 1\n",
    "    ixes.append(ix)\n",
    "  return ixes\n",
    "\n",
    "def clip_diag(w, cval):\n",
    "    n = min(np.shape(w))\n",
    "    w[range(n), range(n)] = np.clip(np.diagonal(w), -cval, cval)\n",
    "\n",
    "n, p = 0, 0\n",
    "mWxh, mWhh, mWhy = np.zeros_like(Wxh), np.zeros_like(Whh), np.zeros_like(Why)\n",
    "mbh, mby = np.zeros_like(bh), np.zeros_like(by) # memory variables for Adagrad\n",
    "smooth_loss = -np.log(1.0/vocab_size)*seq_length # loss at iteration 0\n",
    "#while True:\n",
    "while n<nruns:\n",
    "  # prepare inputs (we're sweeping from left to right in steps seq_length long)\n",
    "  if p+seq_length+1 >= len(data) or n == 0: \n",
    "    hprev = np.zeros((hidden_size,1)) # reset RNN memory\n",
    "    p = 0 # go from start of data\n",
    "  inputs = [char_to_ix[ch] for ch in data[p:p+seq_length]]\n",
    "  targets = [char_to_ix[ch] for ch in data[p+1:p+seq_length+1]]\n",
    "\n",
    "  # sample from the model now and then\n",
    "  if n % noutput == 0:\n",
    "    sample_ix = sample(hprev, inputs[0], 500)\n",
    "    txt = ''.join(ix_to_char[ix] for ix in sample_ix)\n",
    "    print('----\\n %s \\n----' % (txt, ))\n",
    "\n",
    "  # forward seq_length characters through the net and fetch gradient\n",
    "  loss, dWxh, dWhh, dWhy, dbh, dby, hprev = lossFun(inputs, targets, hprev)\n",
    "  smooth_loss = smooth_loss * 0.999 + loss * 0.001\n",
    "  if n % noutput == 0:\n",
    "        print('iter %d, loss: %f' % (n, smooth_loss)) # print progress\n",
    "  \n",
    "  # perform parameter update with Adagrad\n",
    "  for param, dparam, mem in zip([Wxh, Whh, Why, bh, by], \n",
    "                                [dWxh, dWhh, dWhy, dbh, dby], \n",
    "                                [mWxh, mWhh, mWhy, mbh, mby]):\n",
    "    mem += dparam * dparam\n",
    "    param += -learning_rate * dparam / np.sqrt(mem + 1e-8) # adagrad update\n",
    "\n",
    "  p += seq_length # move data pointer\n",
    "  n += 1 # iteration counter "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'pygame'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-28f0dd588c7a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mpygame\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmidi\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mrandom\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mpygame\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmidi\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'pygame'"
     ]
    }
   ],
   "source": [
    "import pygame.midi\n",
    "import time\n",
    "import random\n",
    "\n",
    "pygame.midi.init()\n",
    "player = pygame.midi.Output(0)\n",
    "player.set_instrument(1) # https://pjb.com.au/muscript/gm.html\n",
    "notel = [55, 72, 77, 74, 50, 67, 77, 79, 55, 79, 81, 53, 58, 50, 77, 75, 75, 51, 74, 79, 72, 58, 77, 74, 75, 57, 74, 74, 72, 69, 72, 65, 72, 63, 51, 67, 67, 46, 70, 75, 72, 41, 70, 68, 48, 63, 74, 63, 70, 63, 70, 70, 75, 51, 69, 65, 67, 70, 50, 63, 75, 72, 46, 70, 65, 69, 82, 72, 74, 68, 72, 53, 72, 69, 77, 67, 75, 72, 55, 69, 65, 82, 51, 75, 77, 77, 75, 50, 79, 79, 74, 79, 79, 82, 67, 79, 63, 79, 79, 79, 75, 53, 79, 74, 74, 75, 46, 70, 77, 74, 55, 77, 79, 77, 82, 77, 48, 53, 79, 79, 51, 74, 79, 79]\n",
    "for n in notel:\n",
    "    player.note_on(n, 127) # note_on(note, velocity=None, channel = 0) https://en.scratch-wiki.info/wiki/MIDI_Notes\n",
    "    time.sleep(0.17)\n",
    "    player.note_off(n, 127)\n",
    "del player\n",
    "pygame.midi.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[51, 63, 63, 67, 67, 70, 51, 70, 75, 55, 75, 70, 55, 70, 72, 56, 72, 80, 84, 56, 80, 84, 79, 82, 58, 79, 82, 77, 80, 58, 77, 80, 75, 79, 46, 75, 79, 74, 77, 46, 74, 77, 75, 79, 51, 79, 63, 75, 63, 63, 67, 67, 70, 51, 63, 70, 63, 75, 55, 75, 70, 63, 55, 70, 63, 72, 56, 63, 72, 80, 84, 56, 80, 84, 79, 82, 58, 79, 82, 77, 80, 58, 77, 80, 75, 79, 46, 75, 79, 74, 77, 46, 74, 77, 75, 79, 51, 79, 82, 75, 82, 82, 80, 80, 79, 51, 82, 79, 84, 77, 56, 77, 75, 84, 56, 75, 77, 74, 58, 77, 74, 58, 79, 75, 63, 63, 58, 58, 55, 79, 75, 55, 80, 77, 51, 80, 77, 82, 79, 51, 82, 79, 77, 74, 58, 77, 74, 58, 79, 75, 63, 63, 58, 58, 55, 79, 75, 55, 80, 77, 51, 80, 77, 82, 79, 51, 82, 79, 77, 74, 58, 77, 74, 75, 72, 58, 75, 72, 74, 70, 46, 74, 70, 72, 68, 46, 72, 68, 70, 67, 58, 70, 67, 68, 65, 58, 68, 65, 67, 63, 51, 67, 63, 63, 67, 67, 70, 63, 51, 70, 75, 70, 55, 75, 70, 70, 67, 55, 70, 67, 72, 63, 56, 72, 63, 84, 80, 56, 84, 80, 82, 79, 58, 82, 79, 80, 77, 58, 80, 77, 79, 75, 46, 79, 75, 77, 74, 46, 77, 74, 79, 75, 51, 79, 63, 75, 63, 67, 63, 67, 70, 51, 63, 70, 75, 63, 55, 75, 70, 63, 55, 70, 72, 63, 56, 72, 63, 84, 80, 56, 84, 80, 82, 79, 58, 82, 79, 80, 77, 58, 80, 77, 79, 75, 46, 79, 75, 77, 74, 46, 77, 74, 79, 75, 51, 79, 77, 75, 51, 77, 75, 79, 51, 75, 77, 79, 51, 77, 79, 75, 51, 79, 80, 75, 51, 80, 82, 77, 50, 82, 77, 77, 50, 77, 74, 82, 50, 74, 70, 82, 50, 70, 77, 50, 77, 74, 50, 74, 75, 48, 75, 74, 48, 74, 72, 75, 48, 72, 74, 75, 48, 74, 75, 72, 48, 75, 77, 72, 48, 77, 79, 74, 46, 79, 74, 74, 46, 74, 70, 79, 46, 70, 67, 79, 46, 67, 74, 46, 74, 70, 46, 70, 72, 44, 72, 68, 44, 68, 75, 72, 56, 75, 72, 72, 56, 72, 70, 75, 56, 70, 68, 56, 68, 70, 55, 70, 67, 75, 55, 67, 75, 70, 55, 75, 70, 70, 55, 70, 68, 75, 55, 68, 67, 75, 55, 67, 68, 74, 53, 68, 74, 67, 75, 53, 67, 75, 68, 77, 50, 68, 70, 77, 50, 70, 67, 75, 51, 67, 68, 75, 51, 68, 65, 74, 46, 65, 72, 74, 46, 72, 70, 50, 70, 68, 50, 68, 67, 46, 67, 65, 46, 65, 67, 51, 67, 77, 51, 77, 75, 48, 75, 74, 48, 74, 72, 51, 72, 70, 51, 70, 69, 53, 69, 79, 53, 79, 77, 57, 77, 75, 57, 75, 74, 53, 74, 72, 53, 72, 74, 58, 65, 74, 58, 65, 74, 70, 58, 70, 72, 74, 58, 72, 70, 74, 58, 74, 75, 70, 58, 75, 72, 77, 57, 77, 72, 72, 57, 72, 77, 69, 57, 69, 65, 77, 57, 65, 72, 57, 72, 69, 57, 69, 70, 55, 70, 69, 55, 69, 70, 67, 55, 67, 69, 70, 55, 69, 67, 70, 55, 70, 72, 67, 55, 72, 69, 74, 53, 74, 69, 69, 53, 69, 74, 65, 53, 65, 62, 74, 53, 62, 69, 53, 69, 65, 53, 65, 67, 51, 67, 63, 51, 63, 67, 70, 51, 70, 67, 67, 51, 67, 70, 65, 51, 65, 63, 51, 63, 65, 50, 65, 62, 70, 50, 62, 65, 70, 50, 70, 65, 65, 50, 65, 70, 63, 50, 63, 62, 70, 50, 62, 69, 63, 48, 69, 63, 70, 62, 48, 70, 62, 72, 63, 45, 63, 65, 72, 45, 65, 70, 62, 46, 62, 63, 70, 46, 63, 79, 60, 51, 79, 80, 60, 51, 80, 79, 72, 51, 79, 77, 72, 51, 77, 75, 51, 75, 74, 51, 74, 75, 79, 51, 75, 79, 77, 80, 51, 77, 80, 75, 79, 51, 75, 79, 74, 77, 51, 74, 77, 72, 75, 51, 72, 75, 70, 74, 51, 70, 74, 69, 72, 53, 72, 77, 69, 77, 77, 81, 81, 72, 53, 77, 72, 77, 74, 58, 74, 82, 77, 58, 82, 77, 75, 57, 77, 79, 75, 79, 77, 74, 77, 79, 57, 74, 79, 77, 75, 53, 77, 79, 75, 79, 77, 84, 77, 79, 53, 84, 79, 77, 74, 58, 77, 79, 74, 79, 77, 82, 77, 79, 58, 82, 79, 77, 72, 53, 77, 79, 72, 79, 77, 77, 77, 79, 77, 79, 77, 81, 77, 79, 81, 79, 77, 72, 53, 77, 79, 72, 79, 77, 74, 58, 77, 79, 74, 79, 77, 82, 77, 79, 58, 82, 79, 77, 75, 57, 77, 79, 75, 79, 77, 74, 77, 79, 57, 74, 79, 77, 75, 53, 77, 79, 75, 79, 77, 84, 77, 79, 53, 84, 79, 77, 74, 58, 77, 79, 74, 79, 77, 82, 77, 79, 58, 82, 79, 77, 72, 53, 72, 77, 77, 53, 77, 72, 81, 41, 72, 81, 70, 79, 41, 70, 79, 72, 81, 41, 72, 81, 74, 82, 41, 74, 82, 75, 84, 41, 75, 84, 74, 82, 41, 74, 82, 72, 81, 41, 72, 81, 75, 79, 41, 75, 79, 74, 77, 41, 74, 77, 72, 75, 41, 72, 75, 74, 46, 70, 74, 70, 74, 70, 74, 65, 46, 70, 65, 67, 70, 51, 67, 75, 70, 51, 75, 68, 70, 50, 70, 72, 68, 67, 72, 70, 70, 72, 50, 67, 68, 72, 70, 46, 70, 72, 68, 77, 72, 70, 70, 72, 46, 77, 67, 72, 70, 51, 70, 72, 67, 75, 72, 70, 70, 72, 51, 75, 65, 72, 70, 46, 70, 72, 65, 70, 72, 70, 70, 72, 70, 74, 72, 70, 70, 72, 74, 65, 72, 70, 46, 70, 72, 65, 67, 72, 70, 51, 70, 72, 67, 75, 72, 70, 70, 72, 51, 75, 68, 72, 70, 50, 70, 72, 68, 67, 72, 70, 70, 72, 50, 67, 68, 72, 70, 46, 70, 72, 68, 77, 72, 70, 70, 72, 46, 77, 67, 72, 70, 51, 70, 72, 67, 75, 72, 70, 70, 72, 51, 75, 65, 72, 70, 46, 65, 70, 70, 46, 70, 74, 70, 46, 74, 70, 75, 72, 46, 75, 72, 77, 74, 46, 77, 74, 79, 75, 46, 79, 75, 80, 77, 46, 80, 77, 79, 75, 46, 79, 75, 77, 74, 58, 77, 74, 80, 77, 58, 80, 77, 79, 75, 46, 79, 75, 77, 74, 46, 77, 74, 79, 75, 51, 79, 77, 75, 51, 77, 75, 79, 48, 75, 79, 74, 77, 48, 74, 77, 72, 75, 51, 72, 75, 70, 74, 51, 70, 74, 69, 72, 53, 69, 72, 70, 70, 53, 70, 70, 72, 69, 41, 72, 69, 75, 72, 41, 75, 72, 74, 70, 53, 74, 70, 72, 69, 53, 72, 69, 74, 70, 58, 74, 70, 75, 72, 58, 75, 72, 77, 74, 46, 46, 48, 48, 50, 50, 51, 51, 53, 77, 74, 72, 69, 53, 72, 69, 72, 69, 41, 72, 69, 75, 72, 41, 75, 72, 74, 70, 53, 74, 70, 72, 69, 53, 72, 69, 74, 70, 46, 74, 70, 75, 72, 46, 75, 72, 77, 74, 46, 46, 48, 48, 50, 50, 51, 51, 53, 77, 74, 72, 69, 53, 72, 69, 72, 69, 41, 72, 69, 75, 72, 41, 75, 72, 74, 70, 53, 74, 70, 72, 69, 53, 72, 69, 74, 70, 46, 74, 75, 46, 75, 75, 48, 75, 77, 48, 77, 77, 50, 77, 79, 50, 79, 79, 51, 79, 81, 70, 75, 51, 81, 75, 81, 75, 53, 81, 82, 75, 53, 82, 82, 74, 55, 82, 70, 74, 55, 70, 79, 72, 51, 79, 75, 72, 51, 75, 74, 70, 53, 74, 70, 53, 72, 69, 41, 72, 69, 41, 70, 74, 46, 70, 74, 72, 75, 46, 72, 75, 74, 77, 46, 46, 48, 48, 50, 50, 51, 51, 53, 74, 77, 69, 72, 53, 69, 72, 69, 72, 41, 69, 72, 72, 75, 41, 72, 75, 70, 74, 53, 70, 74, 69, 72, 53, 69, 72, 70, 74, 46, 70, 74, 72, 75, 46, 72, 75, 74, 77, 46, 46, 48, 48, 50, 50, 51, 51, 53, 74, 77, 69, 72, 53, 69, 72, 69, 72, 41, 69, 72, 72, 75, 41, 72, 75, 70, 74, 53, 70, 74, 69, 72, 53, 69, 72, 70, 74, 46, 70, 74, 74, 77, 46, 74, 77, 77, 82, 50, 82, 81, 50, 81, 79, 46, 79, 77, 46, 77, 79, 51, 79, 77, 77, 77, 75, 79, 51, 79, 81, 53, 75, 81, 53, 74, 82, 55, 74, 82, 55, 75, 79, 51, 75, 79, 72, 75, 51, 72, 75, 70, 74, 53, 70, 74, 69, 72, 69, 72, 70, 74, 70, 74, 69, 72, 53, 69, 72, 70, 74, 70, 74, 69, 72, 41, 69, 72, 70, 70, 41, 70, 70, 70, 70, 46, 70, 70, 46, 46, 58, 58, 62, 62, 65, 46, 65, 70, 50, 70, 65, 50, 65, 67, 51, 67, 75, 79, 51, 75, 79, 74, 77, 53, 74, 77, 72, 75, 53, 72, 75, 70, 74, 41, 70, 74, 69, 72, 41, 69, 72, 70, 74, 46, 74, 58, 70, 58, 82, 62, 62, 65, 46, 82, 65, 70, 70, 50, 70, 65, 50, 65, 67, 51, 70, 67, 75, 79, 51, 75, 79, 74, 77, 53, 74, 77, 72, 75, 53, 72, 75, 70, 74, 41, 70, 74, 69, 72, 41, 69, 72, 70, 74, 46, 74, 77, 70, 77, 77, 75, 75, 74, 46, 77, 74, 79, 72, 51, 72, 70, 79, 51, 70, 72, 69, 53, 72, 69, 53, 74, 70, 58, 58, 53, 53, 50, 74, 70, 50, 75, 72, 46, 75, 72, 77, 74, 46, 77, 74, 72, 69, 53, 72, 69, 53, 74, 70, 58, 58, 53, 53, 50, 74, 70, 50, 75, 72, 46, 75, 72, 77, 74, 46, 77, 74, 72, 69, 53, 72, 65, 53, 51, 69, 65, 51, 77, 50, 50, 48, 48, 47, 77, 80, 80, 79, 74, 79, 77, 77, 75, 75, 47, 74, 74, 74, 75, 74, 48, 75, 48, 74, 47, 74, 74, 47, 75, 67, 48, 75, 48, 77, 50, 67, 77, 50, 79, 72, 51, 72, 51, 70, 48, 70, 48, 68, 53, 79, 68, 77, 80, 77, 80, 75, 79, 75, 79, 74, 77, 74, 77, 53, 72, 75, 72, 75, 71, 74, 55, 71, 74, 55, 72, 75, 60, 60, 55, 55, 51, 72, 75, 51, 74, 77, 48, 74, 77, 75, 79, 48, 75, 79, 71, 74, 55, 71, 74, 55, 72, 75, 60, 60, 55, 55, 51, 72, 75, 51, 74, 77, 48, 74, 77, 75, 79, 48, 75, 79, 71, 74, 55, 71, 79, 74, 55, 79, 79, 59, 79, 77, 59, 77, 77, 55, 77, 75, 55, 75, 74, 60, 84, 74, 60, 84, 76, 84, 48, 84, 82, 48, 82, 82, 60, 82, 80, 76, 60, 80, 80, 53, 77, 80, 53, 77, 77, 81, 41, 77, 75, 41, 75, 75, 53, 75, 74, 81, 53, 74, 74, 58, 82, 74, 58, 82, 74, 82, 46, 82, 80, 46, 80, 80, 58, 80, 79, 74, 58, 79, 75, 79, 51, 79, 79, 75, 51, 79, 63, 80, 48, 80, 79, 63, 48, 79, 75, 77, 60, 77, 75, 75, 75, 74, 77, 77, 77, 74, 60, 77, 75, 79, 59, 75, 79, 74, 77, 74, 75, 75, 74, 74, 75, 77, 75, 74, 75, 75, 74, 59, 74, 74, 75, 60, 79, 75, 60, 79, 80, 63, 48, 80, 79, 63, 48, 79, 77, 75, 60, 77, 75, 75, 60, 75, 77, 74, 60, 77, 77, 74, 60, 77, 79, 74, 59, 79, 77, 77, 75, 75, 74, 74, 59, 74, 75, 60, 75, 60, 60, 58, 60, 58, 72, 57, 57, 55, 55, 54, 72, 75, 75, 69, 74, 74, 72, 72, 70, 70, 54, 69, 69, 70, 55, 70, 55, 69, 54, 69, 69, 54, 62, 70, 55, 70, 55, 72, 57, 62, 72, 57, 67, 74, 59, 67, 59, 65, 55, 65, 55, 63, 60, 63, 74, 60, 75, 72, 48, 75, 72, 74, 70, 74, 70, 72, 69, 72, 69, 70, 67, 48, 70, 67, 69, 66, 50, 69, 66, 50, 70, 67, 55, 55, 50, 50, 46, 70, 67, 46, 72, 69, 43, 72, 69, 74, 70, 43, 74, 70, 69, 66, 50, 69, 66, 50, 70, 67, 55, 55, 50, 50, 46, 70, 67, 46, 72, 69, 43, 72, 69, 74, 70, 43, 74, 70, 69, 66, 50, 66, 74, 69, 50, 74, 74, 54, 74, 72, 54, 72, 72, 50, 72, 70, 50, 70, 70, 55, 79, 70, 55, 79, 79, 71, 43, 79, 77, 43, 77, 77, 55, 77, 75, 71, 55, 75, 75, 48, 72, 75, 48, 72, 76, 72, 60, 72, 70, 60, 70, 70, 48, 70, 69, 76, 48, 69, 69, 53, 77, 53, 77, 77, 41, 77, 75, 41, 75, 75, 53, 75, 69, 74, 53, 74, 74, 70, 46, 74, 74, 46, 74, 75, 43, 75, 74, 43, 74, 72, 55, 72, 70, 70, 55, 70, 72, 69, 55, 72, 72, 69, 55, 72, 74, 69, 54, 74, 72, 72, 70, 70, 69, 69, 54, 69, 70, 55, 74, 70, 55, 74, 70, 75, 43, 75, 74, 43, 74, 72, 55, 72, 70, 70, 55, 70, 69, 72, 55, 72, 72, 69, 55, 72, 69, 74, 54, 74, 72, 72, 70, 70, 69, 69, 54, 69, 67, 70, 55, 67, 70, 82, 74, 55, 82, 74, 82, 74, 43, 82, 74, 81, 72, 43, 81, 72, 81, 72, 48, 81, 72, 79, 70, 48, 79, 70, 79, 70, 50, 79, 70, 78, 69, 78, 69, 79, 70, 79, 70, 78, 69, 50, 78, 69, 79, 70, 79, 70, 78, 69, 48, 78, 69, 48, 79, 67, 46, 79, 74, 67, 46, 74, 75, 67, 48, 75, 67, 72, 69, 48, 72, 69, 70, 67, 50, 70, 67, 69, 66, 69, 66, 70, 67, 70, 67, 69, 66, 50, 69, 66, 70, 67, 70, 67, 69, 66, 38, 69, 66, 67, 67, 38, 67, 67, 67, 67, 43, 67, 70, 43, 70, 72, 44, 72, 44, 74, 46, 67, 74, 46, 75, 48, 75, 48, 77, 50, 77, 50, 79, 51, 63, 63, 67, 67, 70, 79, 51, 70, 75, 55, 75, 70, 55, 70, 72, 56, 72, 80, 84, 56, 80, 84, 79, 82, 58, 79, 82, 77, 80, 58, 77, 80, 75, 79, 46, 75, 79, 74, 77, 46, 74, 77, 75, 79, 51, 79, 63, 75, 63, 63, 67, 67, 70, 51, 63, 70, 63, 75, 55, 75, 70, 63, 55, 70, 63, 72, 56, 63, 72, 80, 84, 56, 80, 84, 79, 82, 58, 79, 82, 77, 80, 58, 77, 80, 75, 79, 46, 75, 79, 74, 77, 46, 74, 77, 75, 79, 51, 75, 70, 70, 75, 75, 79, 79, 51, 79, 82, 55, 82, 70, 55, 70, 72, 56, 72, 82, 56, 82, 80, 53, 80, 79, 53, 79, 80, 80, 72, 72, 70, 50, 70, 80, 50, 80, 79, 51, 79, 77, 51, 77, 79, 79, 70, 70, 68, 48, 68, 79, 48, 79, 77, 50, 77, 75, 50, 75, 77, 46, 77, 68, 46, 68, 67, 51, 70, 67, 70, 75, 75, 79, 51, 79, 82, 55, 82, 70, 55, 70, 72, 56, 72, 82, 56, 82, 80, 53, 80, 79, 53, 79, 80, 80, 72, 72, 70, 50, 70, 80, 50, 80, 79, 51, 79, 77, 51, 77, 79, 79, 70, 70, 68, 48, 68, 79, 48, 79, 77, 50, 77, 75, 50, 75, 77, 46, 77, 68, 46, 68, 67, 51, 70, 70, 75, 67, 75, 73, 70, 51, 73, 70, 72, 68, 55, 72, 68, 70, 67, 55, 70, 67, 72, 68, 56, 72, 68, 73, 75, 56, 73, 75, 72, 80, 53, 72, 80, 70, 79, 53, 70, 79, 68, 77, 68, 77, 67, 75, 67, 75, 65, 74, 46, 65, 70, 74, 70, 74, 70, 74, 65, 46, 70, 65, 67, 70, 51, 67, 75, 70, 51, 75, 68, 70, 50, 70, 72, 68, 67, 72, 70, 70, 72, 50, 67, 68, 72, 70, 46, 70, 72, 68, 77, 72, 70, 70, 72, 46, 77, 67, 72, 70, 51, 70, 72, 67, 75, 72, 70, 70, 72, 51, 75, 65, 72, 70, 46, 70, 72, 65, 70, 72, 70, 70, 72, 70, 74, 72, 70, 70, 72, 74, 65, 72, 70, 46, 70, 72, 65, 67, 72, 70, 51, 70, 72, 67, 75, 72, 70, 70, 72, 51, 75, 68, 72, 70, 50, 70, 72, 68, 67, 72, 70, 70, 72, 50, 67, 68, 72, 70, 46, 70, 72, 68, 77, 72, 70, 70, 72, 46, 77, 67, 72, 70, 51, 70, 72, 67, 75, 72, 70, 70, 72, 51, 75, 65, 72, 70, 46, 65, 70, 70, 46, 70, 74, 70, 58, 74, 70, 75, 72, 58, 75, 72, 77, 74, 46, 77, 74, 79, 75, 46, 79, 75, 80, 77, 46, 80, 77, 79, 75, 46, 79, 75, 77, 74, 46, 77, 74, 79, 75, 46, 79, 75, 80, 77, 46, 80, 77, 77, 74, 46, 77, 74, 79, 73, 51, 73, 72, 79, 72, 75, 73, 73, 82, 51, 75, 82, 75, 72, 56, 72, 80, 75, 56, 80, 75, 73, 55, 75, 77, 73, 77, 75, 72, 75, 77, 55, 72, 77, 75, 73, 51, 75, 77, 73, 77, 75, 82, 75, 77, 51, 82, 77, 75, 72, 56, 75, 77, 72, 77, 75, 80, 75, 77, 56, 80, 77, 75, 70, 51, 75, 77, 70, 77, 75, 75, 75, 77]\n"
     ]
    }
   ],
   "source": [
    "# read out the midi file\n",
    "import mido\n",
    "\n",
    "#outport = mido.open_output()\n",
    "niter = 0\n",
    "noteslst = []\n",
    "for msg in mido.MidiFile('bjs1031c.mid').play():\n",
    "    #outport.send(msg)\n",
    "    #print msg.bytes()\n",
    "    #print str(msg)\n",
    "    thismsg = str(msg).split(\" \") # ['note_on', 'channel=0', 'note=60', 'velocity=61', 'time=0.00168918958333']\n",
    "    # The velocity specifies the volume or force, with which the note is played\n",
    "    if thismsg[0] == 'note_on':\n",
    "        #print thismsg[2][5:]\n",
    "        noteslst.append(int(thismsg[2][5:]))\n",
    "    niter += 1\n",
    "    if niter >= 3000:\n",
    "        break\n",
    "print(noteslst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data has 36979 characters, 14 unique.\n",
      "----\n",
      " 95170203659:4'::0 ,2',8',880694 69'3' 0222'27'247:287 ,'10:06'58'9,2'2,0256'210'057818:399322180739269 4 2 11711990,29'9, 648::12:20,68527  ,2907154:445095532,642:85355:2521:0,5103'66:5,4:41887199681 '5,,876 12449,41::''82''07 ,   306  1,96'7:1',8201622883 841':1391:285:0,,37 7,176:, 196859 '6, 29'959'77'8844',, 7 2041:,5738420'9494',80,24'9'041''52,10,''9:6'051355''1465,: :390 '088635,':9,8 '6,11217407:10 1,11957'197'1''95:0',7,:551 :36:,'57'87' :,07323012215 55:7'1',4127: :12:020  1: 7213601 144:'60,7049',''4:61891 6,4650 :'',06,1'1': 626:0849967943524 6:,2:3276:494753:9476,8'749 3 6:,9882:5860799:' 1,216'49113 54796:24:113135:,3:'838480,459,5632,10 ,25'3'72,57:97577'630:788 95,6 162:68,67299':64 ,' 609081164573 '806878682'7206525 5 :390684,267936 :58 '952,72,6,1536 , :60,751',245563 '9378,:5':716: 4261::2 2,'902,75:2 '0938'19499,91:985:'3'796307288729:94''0 812372, 508,819024 7235496,57, 9060:2,9,8 35714916,21''991 0,494478 04 9'2'047'242:8 4'64265:16,157825 572'0 0,'6,88 24791  3'2 42332995 7 928'17 0483'26':51:4212184,5'' 23'21,1,305695,8:':9',8 22'05, 59190193459802,,630,1684789539 9'4 '32'82905'41:7944'70 365':64,,' ,4'2166548695311226 ':8505,45' 231'3036918716,35 9:5,:2,428 07 '915:4,86'2,29:899 8:6589323'::0:7851446,8,3 703 :2800:48' 086 637,6:,55,'583,60379'7,92 41,::9'40:97639148,4'11212068'3050''6,28:4', 785,745384657384,23 :3,1763'0',:4'17 67,90967:340:44148366,::88:'04:3': 951,206863:5'58 20,821294342:71 :34'7:2876:81188,79'6 08'2 ,66:03,':883:1498''516 0,471'40 \n",
      "----\n",
      "iter 0, loss: 65.976428\n",
      "----\n",
      " 80', '2:46:80', '1:79:80', '2:53:80', '0:69:0', '0:77:80', '2:45:0', '0:74:80', '2:46:0', '0:74:80', '2:51:80', '2:56:80', '1:70:80', '2:53:80', '0:75:80', '2:41:0', '1:70:80', '1:73:0', '0:82:0', '0:72:80', '1:70:80', '2:51:80', '1:79:80', '1:74:80', '2:50:0', '0:75:80', '0:82:0', '1:74:0', '2:53:80', '0:74:80', '2:46:0', '2:46:0', '0:72:0', '1:71:0', '0:72:80', '0:75:0', '0:82:80', '2:46:0', '0:77:80', '2:51:80', '2:75:0', '0:73:80', '0:75:80', '1:70:0', '0:69:0', '2:55:0', '1:75:0', '0:75:80', '1:82:80', '1:70:80', '0:70:0', '0:62:0', '0:72:0', '0:75:0', '0:82:0', '0:79:80', '2:51:0', '0:79:80', '1:74:0', '1:65:80', '2:51:80', '0:73:0', '1:74:80', '2:46:0', '1:75:0', '0:70:80'0:62:80', '2:56:0', '1:77:0', '0:75:80', '1:75:0', '0:72:0', '0:75:80', '2:53:80', 61:75:80', '2:53:80', '0:72:80', '2:46:80', '1:72:0', '0:70:0', '0:75:0', '0:75:80', '1:77:0', '0:72:0', '0:75:80', '1:75:0', '1:74:80', '2:41:0', '1:75:80', '2:58:0', '1:62:80', '1:79:80', '2:51:80', '2:43:0', '0:70:80', '1:75:80', '2:51:80', '1:75:0', '0:72:0', '0:69:0', '0:79:80', '1:79:0', '1:75:0', '0:75:0', '0:74:0', '1:79:0', '1:79:0', '0:77:80', '2:57:0', '0:75:80', '2:41:80', '0:74:0', '1:75:80', '2:58:0', '0:72:80', '2:48:0', '1:69:80', '2:46:0', '0:74:80', '1:75:80', '1:75:80', '0:74:0', '1:84:80', '2:48:80', '2:58:0', '0:72:0', '1:79:80', '1:73:0', '0:80:0', '1:70:80', '2:58:0', '0:79:80', '2:53:80', '0:74:80', '2:46:80', '1:75:0', '0:74:80', '1:80:80', '2:57:80', '0:72:0', '0:63:0', '1:72:80', '2:46:0', '0: \n",
      "----\n",
      "iter 5000, loss: 10.806418\n",
      "----\n",
      "  '2:56:80', '0:72:0', '1:72:84', '1:77:80', '0:70:0', '1:69:0', '1:80:0', '0:70:80', '1:74:80', '2:46:0', '1:51:0', '0:79:80', '2:46:80', '1:70:80', '0:70:0', '1:69:80', '1:70:80', '0:79:80', '0:75:0', '0:80:80', '0:70:80', '1:75:0', '1:79:0', '1:72:80', '2:74:0', '1:82:0', '0:74:0', '1:69:80', '1:72:80', '1:75:80', '1:77:0', '1:77:0', '1:70:80', '2:50:0', '1:79:80', '2:51:0', '1:66:0', '0:72:0', '0:80:0', '1:75:80', '1:76:80', '2:50:0', '2:51:80', '0:72:80', '1:70:80', '0:74:80', '0:77:0', '0:75:0', '2:45:80', '0:74:0', '1:80:80', '0:82:0', '2:50:0', '2:48:0', '2:54:80', '2:60:0', '2:50:80', '0:70:80', '1:69:80', '2:50:0', '1:72:0', '1:77:0', '0:75:0', '1:80:0', '0:70:80', '0:80:0', '2:58:80', '1:67:80', '0:75:80', '2:58:80', '2:56:80', '0:80:80', '2:50:0', '1:70:0', '0:77:80', '0:72:80', '2:55:0':80', '2:58:80', '1:74:0', '1:80:80', '1:70:0', '1:67:0', '1:80:0', '2:46:80', '7:72:0', '0:74:0', '0:80:80', '1:65:0', '0:77:80', '2:51:0', '1:77:0', '1:68:0', '2:53:0', '1:65:0', '1:70:80', '0:77:80', '1:80:0', '1:72:0', '2:51:80', '2:58:80', '2:43:0', '0:75:80', '1:70:80', '2:58:80', '2:53:80', '1:75:80', '2:51:0', '2:51:80', '1:79:80', '1:68:0', '0:75:80', '1:77:80', '1:70:80', '0:72:80', '0:63:0', '0:80:80', '1:70:80', '2:57:80', '2:51:0', '2:48:80', '1:72:0', '0:74:0', '1:64:0', '1:68:0', '1:69:0', '1:70:80', '2:58:0', '1:69:0', '0:79:0', '2:46:80', '2:41:80', '1:77:80', '1:70:0', '1:76:80', '0:79:0', '1:75:0', '0:67:80', '1:80:80', '2:46:0', '1:66:0', '1:72:80', '2:51:0', '0: \n",
      "----\n",
      "iter 10000, loss: 10.424420\n",
      "----\n",
      " 0:75:0', '0:68:80', '2:48:80', '2:53:0', '0:72:80', '2:46:0', '0:70:80', '2:46:80', '0:75:0', '2:68:80', '2:58:80', '0:77:0', '0:77:0', '0:66:80', '0:77:0', '0:79:80', '2:46:80', '0:69:0', '0:79:80', '0:79:0', '0:70:0', '0:80:0', '0:82:0', '1:74:80', '1:68:0', '0:77:0', '0:67:0', '0:78:0', '0:74:0', '0:63:80', '0:80:0', '0:70:0', '1:72:0', '0:75:0', '0:67:0', '0:65:0', '2:50:0', '0:74:0', '0:68:0', '0:85:0', '0:70:80', '2:46:80', '2:51:0', '2:41:0', '2:56:0', '0:72:0', '1:75:0', '0:74:0', '0:65:80', '1:68:0', '0:77:0', '0:63:0', '2:55:0', '1:75:80', '2:46:80', '0:75:0', '1:70:80', '1:80:0', '0:70:80', '0:70:0', '0:72:0', '0:74:80', '0:80:0', '0:68:80', '2:46:0', '0:74:00', '0:76:0', '0:72:80', '2:53:80', '0:72:20', '1:84:0', '0:80:0', '0:82:0', '0:67:80', '0:80:0', '0:70:0', '0:81:0', '2:48:80', '0:69:80', '1:74:80', '1:75:0', '2:50:0', '1:70:80', '0:75:0', '0:75:0', '0:82:0', '2:58:0', '2:45:0', '0:80:0', '2:54:80', '0:69:0', '0:69:0', '0:82:80', '2:51:80', '2:55:80', '2:50:0', '0:80:0', '0:72:0', '0:65:80', '1:80:80', '1:70:80', '2:48:80', '1:68:80', '1:54:0', '2:48:0', '0:65:80', '2:51:80', '2:55:0', '1:75:80', '0:75:80', '2:46:0', '0:63:0', '2:53:09:0'0:80:80', '2:48:0', '0:71:0', '0:80:80', '2:63:80', '0:72:0', '2:51:80', '0:79:80', '2:48:80', '0:63:80', '0:75:0', '0:75:0', '0:74:0', '2:54:80', '0:79:80', '0:70:0', '1:77:0', '0:62:80', '2:46:80', '1:75:0', '2:58:80', '0:87:80', '2:51:80', '0:80:0', '0:70:80', '1:74:0', '2:48:80', '0:69:0', '0:65:0', '0:80:0', '0:65:0', ' \n",
      "----\n",
      "iter 15000, loss: 9.792461\n",
      "----\n",
      " :80', '1:68:0', '1:74:0', '2:51:0', '0:79:80', '1:77:0', '2:55:0', '1:79:0', '2:55:0', '0:77:0', '0:77:0', '0:67:0', '2:48:0', '2:53:0', '2:48:0', '1:77:0', '2:55:0', '1:74:80', '1:70:0', '1:77:0', '1:79:80', '2:53:0', '2:55:0', '1:70:0', '0:77:80', '0:70:0', '1:67:0', '1:80:0', '0:80:80', '1:75:84', '2:46:80', '0:74:0', '0:82:80', '1:69:80', '2:53:80', '0:70:0', '1:74:0', '2:51:80', '1:70:0', '0:72:80', '1:72:0', '1:77:0', '0:75:80', '1:75:80', '2:41:80', '0:75:0', '0:75:80', '2:55:0', '0:79:80', '1:70:0', '0:74:0', '1:75:80', '0:79:0', '0:75:80', '2:46:80', '2:58:80', '2:51:0', '1:75:0', '1:75:0', '2:48:0', '2:54:0', '2:48:0', '2:55:0', '0:69:80', '0:72:80', '2:42:0', '2:55:80', '0:72:80', '1:75:0', '2:58:0', '0:77:80', '1:72:80', '2:51:0', '0:75:0', '1:70:0', '0:77:80', '1:74:80', '2:46:80', '0:65:0', '1:74:0', '0:74:80', '1:70:80', '2:41:0', '0:75:0', '0:79:80', '1:72:80', '2:50:80', '2:55:80', '1:74:80', '2:53:0', '0:70:0', '1:74:0', '2:55:80', '0:70:80', '1:63:80', '2:48:80', '0:79:0', '1:75:0', '2:55:0', '0:75:80', '0:67:80', '2:53:0', '0:79:80', '1:79:0', '2:53:0', '0:67:80', '0:65:0', '2:45:0', '0:69:80', '1:74:80', '2:53:80', '0:70:0', '0:69:80', '1:72:80', '2:46:80', '0:75:0', '1:77:0', '0:70:80', '2:58:0', '1:72:0', '0:75:0', '0:67:0', '0:79:0', '0:75:80', '1:77:0', '1:79:0', '2:51:0', '0:74:80', '2:54:80', '2:35:80', '2:46:0', '0:74:80', '1:70:80', '2:51:0', '0:71:0', '1:72:0', '0:79:80', '1:72:80', '2:48:80', '0:74:0', '1:72:0', '0:72:80', '1:77:80', '2:55:0', ' \n",
      "----\n",
      "iter 20000, loss: 9.003997\n",
      "----\n",
      " '1:84:0', '0:70:0', '1:75:80', '1:84:0',8'0:75:80', '2:56:80', '0:65:80', '1:72:80', '0:69:80', '2:50:80', '1:70:80', '1:77:0', '1:68:0', '0:80:0', '1:75:0', '0:74:80', '2:51:80', '1:80:80', '1:69:80', '0:72:80', '6:69:80', '1:70:0', '1:75:80', '2:55:0', '0:67:0', '1:75:0', '0:65:80', '0:75:0', '1:72:0', '0:80:80', '2:51:0', '1:77:80', '0:77:0', '1:87:80', '2:58:0', '1:79:0', '0:75:80', '1:74:80', '0:70:80', '0:75:80', '1:75:0', '0:75:0', '1:72:0', '0:77:0', '2:51:80', '1:80:0', '1:77:80', '2:57:0', '1:80:80', '0:75:80', '1:84:0', '1:67:0', '0:72:80', '2:46:80', '2:46:80', '1:75:0', '1:79:80', '2:55:0', '1:75:70', '2:55:80', '1:79:80', '0 77:0', '1:74:80', '0:75:0', '0:79:0', '0:72:0', '0:75:80', '1:82:0', '1:77:0', '1:62:80', '0:68:0', '0:77:0', '1:77:0', '0:75:0', '1:75:80', '1:80:0', '1:74:80', '0:75:80', '2:50:80', '1:75:80', '1:79:0', '1:82:80', '1:77:0', '0:75:0', '1:75:0', '0:70:0', '0:82:80', '1:77:80', '2:53:0', '0:70:0', '0:82:80', '1:65:0', '0:79:80', '0:77:0', '0:79:0', '1:70:0', '0:74:80', '1:77:0', '0:74:80', '0:75:0', '0:72:0', '1:74:80', '1:75:80', '0:72:0', '0:80:0', '0:75:0', '0:75:80', '1:84:0', '2:46:80', '1:74:80', '1:75:80', '1:66:0', '0:72:0', '1:65:80', '1:75:80', '2:46:80', '0:77:80', '0:72:80', '1:75:80', '1:80:80', '1:64:0', '0:80:0', '0:75:0', '1:67:0', '0:70:80', '2:69:80', '1:75:0', '0:75:0', '1:85:0', '2:55:0', '0:82:80', '1:72:80', '0:74:80', '2:53:80', '0:70:80', '2:48:0', '1:65:0', '1:77:0', '0:79:80', '1:75:80', '2:58:0', '1:75:80', '1:74:80 \n",
      "----\n",
      "iter 25000, loss: 9.500158\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----\n",
      " '1:70:0', '1:74:0', '1:75:0', '0:75:0', '0:79:80', '1:70:80', '2:50:0', '0:74:0', '0:79:0', '1:62:0', '0:77:0', '1:72:0', '0:72:80', '1:72:0', '0:72:80', '0:72:80', '2:55:0', '1:77:0', '0:79:80', '1:63:80', '1:79:80', '2:43:80', '0:72:0', '1:75:80', '0:65:0', '2:50:80', '1:75:80', '1:82:80', '2:53:80', '0:74:0', '1:70:0', '0:74:0', '2:51:80', '0:79:80', '1:72:80', '2:51:80:80', '1:75:80', '2:53:80', '0:77:80', '2:51:80', '0:75:80', '1:82:80', '2:58:80', '0:79:0', '1:74:0', '0:74:0', '0:79:0', '0:70:80', '1:79:80', '0:77:80', '1:72:80', '1:68:0', '0:72:0', '1:74:0', '2:46:80', '0:77:80', '2:51:0', '1:77:0', '2:50:80', '1:74:0', '0:70:0', '2:51:0', '0:70:80', '1:74:0', '1:61:80', '2:50:80', '0:79:0', '0:74:0', '1:70:0', '1:72:0', '0:69:80', '2:56:0', '2:55:0', '0:70:80', '1:65:80', '0:81:80', '0:70:0', '1:81:0', '1:73:80', '1:63:0', '1:70:0', '1:79:80', '2:46:80', '0:70:80', '1:74:80', '2:55:0', '0:68:0', '0:79:80', '2:51:80', '2:50:80', '2:51:0', '2:53:0', '0:77:80', '0:72:80', '1:69:80', '2:41:80', '0:75:80', '0:75:0', '0:67:80', '1:65:0', '0:68:80', '2:46:80', '0:75:80', '1:70:80', '2:48:80', '0:62:80', '2:51:80', '1:70:0', '1:72:0', '0:75:0', '1:65:80', '2:58:80', '1:63:0', '1:70:80', '2:58:80', '0:79:80', '0:74:0', '0:60:80', '2:51:0', '0:72:0', '0:75:80', '2:50:80', '1:77:0', '1:70:80', '2:46:0', '1:70:80', '1:67:0', '0:68:0', '1:72:0', '1:72:80', '1:70:80', '1:63:0', '1:72:80', '1:74:0', '0:82:0', '0:67:80', '1:72:80', '2:50:80', '2:57:0', '1:82:0', '0:80:0', '2:58:80',  \n",
      "----\n",
      "iter 30000, loss: 9.974450\n",
      "----\n",
      " :80', '0:82:80', '2:48:0', '0:72:80', '2:50:0', '0:70:80', '0:72:80', '0:74:0', '2:53:0', '1:70:80', '2:50:80', '2:53:0', '0:77:0', '1:75:80', '2:48:80', '1:75:80', '2:69:80', '1:67:80', '2:48:0', '1:69:0', '1:72:80', '0:84:0', '0:74:80', '2:55:80', '1:70:0', '1:72:0', '1:74:80', '0:74:0', '0:72:80', '0:60:80', '0:69:0', '1:74:80', '0:69:0', '1:80:80', '1:62:0', '1:74:80', '2:51:0', '1:77:0', '1:70:0', '1:74:0', '0:75:80', '1:60:0', '0:79:80', '1:67:80', '1:72:80', '0:67:80', '2:48:80', '1:75:80', '0:74:80'2:50:0', '0:74:0', '0:72:80', '2:45:0', '2:62:80', '0:72:80', '2:74:0', '0:69:0', '0:79:0', '2:50:80', '1:83:0', '1:62:80', '1:70:0', '0:71:0', '0:69:80', '0:77:80', '2:50:0', '2:43:80', '0:70:80', '2:44:0', '1:70:0', '1:75:80', '2:46:80', '1:82:80', '1:65:80', '1:70:0', '0:70:0', '0:77:0', '0:79:80', '1:46:0', '2:48:80', '0:77:0', '0:89:0', '1:77:80', '0:69:80', '2:51:80', '0:74:80', '1:62:0', '0:70:80', '0:74:0', '1:72:0', '1:74:80', '0:75:80', '1:72:0', '0:79:80', '2:46:80', '0:77:0', '0:75:80', '1:70:0', '1:77:0', '1:79:0', '1:79:0', '2:55:0', '0:69:80', '1:69:80', '0:77:0', '1:72:0', '0:77:80', '1:70:0', '1:62:0', '0:70:0', '1:70:0', '1:72:80', '0:70:80', '0:74:0', '1:63:0', '1:60:80', '2:56:0', '0:75:0', '1:75:80', '1:74:80', '1:79:0', '0:70:80', '2:48:0', '0:77:80', '1:69:80', '2:43:0', '0:74:80', '0:77:0', '0:75:80', '1:74:80', '0:69:0', '1:74:0', '0:74:0', '1:82:0', '1:67:80', '0:70:80', '2:45:0', '2:55:0', '0:70:80', '1:72:0', '1:72:0', '2:55:0', '1:62:80', '1:63: \n",
      "----\n",
      "iter 35000, loss: 9.866267\n",
      "----\n",
      " 0', '2:58:80', '2:58:80', '0:69:0', '2:51:80', '2:50:80', '2:46:80', '1:74:80', '2:41:0', '0:70:0', '1:63:80', '2:46:80', '0:72:0', '1:77:80', '1:75:80', '2:53:80', '0:63:80', '2:48:0', '0:69:80', '2:58:80', '2:58:0', '1:77:80', '1:74:80', '0:69:0', '0:74:0', '1:75:0', '1:63:0', '2:51:0', '0:75:0', '0:65:0', '1:72:80', '1:70:80', '2:53:80', '2:58:0', '1:63:80', '2:56:80', '0:82:0', '1:80:80', '2:53:80', '0:63:0', '1:70:80', '1:74:80', '2:46:0', '1:81:0', '1:65:0', '1:79:80', '2:58:0', '0:65:80', '2:50:80', '1:65:0', '0:74:0', '1:77:0', '0:70:80', '0:84:80', '2:53:80', '0:68:0', '2:51:80', '0:77:80', '1:79:0', '2:51:0', '0:77:0', '1:70:80', '2:55:0', '0:64:80', '1:75:80', '2:51:0', '0:82:80', '2:55:0', '0:75:80', '1:63:80', '2:58:80', '2:47:0', '0:77:80', '2:51:0', '0:74:80', '1:74:80', '2:46:80', '2:48:0', '2:55:0', '0:77:0', '1:69:0', '0:77:0', '1:79:80', '1:70:0', '1:70:0', '0:75:80', '1:77:0', '0:77:0', '0:82:0', '1:65:0', '1:77:0', '0:60:80', '1:70:80', '2:51:0', '0:77:80', '1:72:80', '2:51:80', '2:41:0', '1:72:0', '0:72:80', '1:72:0', '2:,8:80', '2:51:80', '1:79:0', '1:66:80', '2:55:80', '0:79:0', '1:70:0', '0:79:0', '1:74:0', '0:67:0', '0:67:80', '2:58:80', '0:69:80', '1:77:80', '1:75:80', '2:53:80', '0:82:0', '0:79:0', '1:65:80', '2:46:0', '0:77:80', '2:46:0', '0:75:80', '2:48:80', '0:80:0', '0:72:80', '1:62:80', '2:53:0', '0:67:0', '0:75:0', '2:58:0', '0:79:80', '2:51:0', '1:62:0', '0:80:0', '1:67:0', '0:74:80', '1:75:80', '2:51:80', '0:70:0', '2:56:80', '0:77:0', '1: \n",
      "----\n",
      "iter 40000, loss: 9.671485\n",
      "----\n",
      " '0:75:0', '0:72:80', '2:58:0', '0:75:0', '0:77:80', '1:79:80', '1:63:80', '2:56:0', '1:72:0', '0:75:0', '0:74:80', '1:80:0', '1:70:0', '1:62:0', '0:74:80', '1:74:80', '2:41:0', '0:75:0', '0:75:80', '1:80:0', '1:73:0', '0:75:0', '1:69:80', '1:87:80', '2:57:0', '1:82:0', '1:70:0', '1:74:80', '2:43:80', '1:74:0', '0:79:80', '1:65:0', '0:79:0', '0:75:80', '1:77:0', '1:74:0', '1:74:0', '1:79:0', '0:72:80', '1:77:80', '2:46:80', '0:75:0', '2:53:80', '2:55:0', '0:77:0', '1:70:80', '2:59:0', '0:72:0', '1:72:0', '1:74:0', '0:74:80', '1:74:80', '2:46:80', '0:75:0', '1:77:80', '2:55:0', '2:56:80', '2:58:0', '1:70:0', '0:72:80', '1:79:80', '2:53:80', '1:77:0', '1:77:0', '1:72:80', '1:65:80', '2:46:0', '0:79:0', '1:74:80', '2:50:80', '2:55:0', '1:77:0', '0:84:0', '0:79:80', '1:74:80', '2:46:80', '2:46:80', '2:51:0', '1:79:80', '2:56:0', '1:79:80', '2:51:80', '0:75:0', '1:72:0', '0:75:80', '1:77:80', '2:43:80', '1:74:0', '1:80:0', '1:70:0', '0:77:0', '1:69:80', '1:75:0', '2:51:0', '1:75:0', '0:79:80', '1:74:80', '2:46:80', '0:72:0', '1:63:80', '1:74:0', '2:51:80', '1:79:0', '1:74:0', '0:84:80', '1:67:80', '1:70:80', '2:53:80', '1:43:0', '1:67:0', '0:67:80', '1:84:0', '1:82:0', '0:70:80', '1:61:80', '2:56:80', '1:72:0', '1:77:80', '0:65:0', '2:46:0', '1:81:0', '1:75:0', '1:79:80', '1:85:80', '2:56:0', '1:72:80', '1:77:0', '2:55:80', '2:46:80', '1:79:80', '2:45:80', '0:75:80', '1:77:80', '2:55:80', '0:69:0', '1:74:0', '2:51:0', '0:72:80', '1:74:0', '0:74:80', '2:46:80', '0:72:80', '1:79:80', \n",
      "----\n",
      "iter 45000, loss: 9.125453\n",
      "----\n",
      " 1:75:80', '0:70:0', '2:51:0', '0:77:80', '2:50:80', '1:67:0', '2:51:0', '0:74:80', '1:60:0', '1:72:80', '2:53:0', '0:70:0', '0:72:0', '0:67:80', '0:70:0', '0:70:80', '1:70:0', '2:50:0', '1:72:80', '0:70:0', '1:70:0', '0:70:0', '1:68:0', '0:72:80', '2:55:80', '2:51:0', '0:77:80', '0:77:0', '0:70:0', '0:72:80', '2:51:0', '1:67:0', '2:48:80', '1:60:0', '0:78:80', '0:70:0', '0:79:80', '0:70:0', '0:67:80', '2:51:0', '0:77:80', '2:50:0', '0:70:80', '0:70:0', '0:67:0', '0:75:80', '0:70:0', '0:72:80', '1:72:0', '0:79:80', '0:72:0', '0:72:80', '1:70:0', '1:72:0', '1:72:80', '0:74:0', '1:60:0', '0:70:0', '0:72:80', '1:72:0', '1:72:80', '2:53:80', '0:67:0', '0:65:80', '0:67:0', '1:74: ', '0:72:80', '0:70:0', '0:70:80', '0:72:0', '1:63:80', '1:68:0', '0:72:80', '0:74:0', '1:70:0', '0:70:80', '2:58:0', '0:72:0', '0:70:80', '2:53:80', '1:72:0', '1:70:80', '2:48:80', '1:70:0', '1:70:80', '1:72:0', '1:70:0', '1:70:80', '1:70:0', '1:72:80', '1:70:0', '1:67:80', '1:72:0', '1:72:80', '1:72:0', '1:70:80', '1:72:0', '1:70:80', '0:74:0', '0:75:0', '0:72:80', '0:72:0', '0:72:0', '1:63:0', '0:72:80', '1:63:0', '2:51:80', '0:72:0', '0:72:80', '1:70:0', '1:70:80', '2:50:80', '0:60:0', '0:67:0', '1:70:80', '1:72:0', '2:46:80', '2:50:80', '0:67:0', '1:77:0', '0:70:80', '2:51:80', '0:70:0', '1:67:0', '0:65:80', '1:70:0', '0:64:80', '2:50:0', '0:72:80', '0:70:0', '2:50:0', '0:72:80', '2:50:0', '1:70:0', '0:70:80', '2:50:80', '0:70:0', '1:67:0', '1:77:0', '0:72:80', '0:70:0', '0:72:80', '1:72:0', '1:70:80' \n",
      "----\n"
     ]
    }
   ],
   "source": [
    "# from https://gist.github.com/karpathy/d4dee566867f8291f086\n",
    "\n",
    "\"\"\"\n",
    "Minimal character-level Vanilla RNN model. Written by Andrej Karpathy (@karpathy)\n",
    "BSD License\n",
    "\"\"\"\n",
    "import numpy as np\n",
    "\n",
    "nruns = 50000\n",
    "noutput = 5000\n",
    "\n",
    "# data I/O\n",
    "data = open('notes0.txt', 'r').read() # should be simple plain text file\n",
    "chars = list(set(data))\n",
    "data_size, vocab_size = len(data), len(chars)\n",
    "print('data has %d characters, %d unique.' % (data_size, vocab_size))\n",
    "char_to_ix = { ch:i for i,ch in enumerate(chars) }\n",
    "ix_to_char = { i:ch for i,ch in enumerate(chars) }\n",
    "\n",
    "# hyperparameters\n",
    "hidden_size = 100 # size of hidden layer of neurons\n",
    "seq_length = 25 # number of steps to unroll the RNN for\n",
    "learning_rate = 1e-1\n",
    "\n",
    "# model parameters\n",
    "Wxh = np.random.randn(hidden_size, vocab_size)*0.01 # input to hidden\n",
    "Whh = np.random.randn(hidden_size, hidden_size)*0.01 # hidden to hidden\n",
    "Why = np.random.randn(vocab_size, hidden_size)*0.01 # hidden to output\n",
    "bh = np.zeros((hidden_size, 1)) # hidden bias\n",
    "by = np.zeros((vocab_size, 1)) # output bias\n",
    "\n",
    "def lossFun(inputs, targets, hprev):\n",
    "  \"\"\"\n",
    "  inputs,targets are both list of integers.\n",
    "  hprev is Hx1 array of initial hidden state\n",
    "  returns the loss, gradients on model parameters, and last hidden state\n",
    "  \"\"\"\n",
    "  xs, hs, ys, ps = {}, {}, {}, {}\n",
    "  hs[-1] = np.copy(hprev)\n",
    "  loss = 0\n",
    "  # forward pass\n",
    "  for t in range(len(inputs)):\n",
    "    xs[t] = np.zeros((vocab_size,1)) # encode in 1-of-k representation\n",
    "    xs[t][inputs[t]] = 1\n",
    "    hs[t] = np.tanh(np.dot(Wxh, xs[t]) + np.dot(Whh, hs[t-1]) + bh) # hidden state\n",
    "    ys[t] = np.dot(Why, hs[t]) + by # unnormalized log probabilities for next chars\n",
    "    ps[t] = np.exp(ys[t]) / np.sum(np.exp(ys[t])) # probabilities for next chars\n",
    "    loss += -np.log(ps[t][targets[t],0]) # softmax (cross-entropy loss)\n",
    "  # backward pass: compute gradients going backwards\n",
    "  dWxh, dWhh, dWhy = np.zeros_like(Wxh), np.zeros_like(Whh), np.zeros_like(Why)\n",
    "  dbh, dby = np.zeros_like(bh), np.zeros_like(by)\n",
    "  dhnext = np.zeros_like(hs[0])\n",
    "  for t in reversed(range(len(inputs))):\n",
    "    dy = np.copy(ps[t])\n",
    "    dy[targets[t]] -= 1 # backprop into y. see http://cs231n.github.io/neural-networks-case-study/#grad if confused here\n",
    "    dWhy += np.dot(dy, hs[t].T)\n",
    "    dby += dy\n",
    "    dh = np.dot(Why.T, dy) + dhnext # backprop into h\n",
    "    dhraw = (1 - hs[t] * hs[t]) * dh # backprop through tanh nonlinearity\n",
    "    dbh += dhraw\n",
    "    dWxh += np.dot(dhraw, xs[t].T)\n",
    "    dWhh += np.dot(dhraw, hs[t-1].T)\n",
    "    dhnext = np.dot(Whh.T, dhraw)\n",
    "  for dparam in [dWxh, dWhh, dWhy, dbh, dby]:\n",
    "    np.clip(dparam, -5, 5, out=dparam) # clip to mitigate exploding gradients\n",
    "    clip_diag(dparam, 1)\n",
    "  return loss, dWxh, dWhh, dWhy, dbh, dby, hs[len(inputs)-1]\n",
    "\n",
    "def sample(h, seed_ix, n):\n",
    "  \"\"\" \n",
    "  sample a sequence of integers from the model \n",
    "  h is memory state, seed_ix is seed letter for first time step\n",
    "  \"\"\"\n",
    "  x = np.zeros((vocab_size, 1))\n",
    "  x[seed_ix] = 1\n",
    "  ixes = []\n",
    "  for t in range(n):\n",
    "    h = np.tanh(np.dot(Wxh, x) + np.dot(Whh, h) + bh)\n",
    "    y = np.dot(Why, h) + by\n",
    "    p = np.exp(y) / np.sum(np.exp(y))\n",
    "    ix = np.random.choice(range(vocab_size), p=p.ravel())\n",
    "    x = np.zeros((vocab_size, 1))\n",
    "    x[ix] = 1\n",
    "    ixes.append(ix)\n",
    "  return ixes\n",
    "\n",
    "def clip_diag(w, cval):\n",
    "    n = min(np.shape(w))\n",
    "    w[range(n), range(n)] = np.clip(np.diagonal(w), -cval, cval)\n",
    "\n",
    "n, p = 0, 0\n",
    "mWxh, mWhh, mWhy = np.zeros_like(Wxh), np.zeros_like(Whh), np.zeros_like(Why)\n",
    "mbh, mby = np.zeros_like(bh), np.zeros_like(by) # memory variables for Adagrad\n",
    "smooth_loss = -np.log(1.0/vocab_size)*seq_length # loss at iteration 0\n",
    "#while True:\n",
    "while n<nruns:\n",
    "  # prepare inputs (we're sweeping from left to right in steps seq_length long)\n",
    "  if p+seq_length+1 >= len(data) or n == 0: \n",
    "    hprev = np.zeros((hidden_size,1)) # reset RNN memory\n",
    "    p = 0 # go from start of data\n",
    "  inputs = [char_to_ix[ch] for ch in data[p:p+seq_length]]\n",
    "  targets = [char_to_ix[ch] for ch in data[p+1:p+seq_length+1]]\n",
    "\n",
    "  # sample from the model now and then\n",
    "  if n % noutput == 0 or n == nruns-1:\n",
    "    sample_ix = sample(hprev, inputs[0], 1500)\n",
    "    txt = ''.join(ix_to_char[ix] for ix in sample_ix)\n",
    "    print('----\\n %s \\n----' % (txt, ))\n",
    "\n",
    "  # forward seq_length characters through the net and fetch gradient\n",
    "  loss, dWxh, dWhh, dWhy, dbh, dby, hprev = lossFun(inputs, targets, hprev)\n",
    "  smooth_loss = smooth_loss * 0.999 + loss * 0.001\n",
    "  if n % noutput == 0:\n",
    "        print('iter %d, loss: %f' % (n, smooth_loss)) # print progress\n",
    "  \n",
    "  # perform parameter update with Adagrad\n",
    "  for param, dparam, mem in zip([Wxh, Whh, Why, bh, by], \n",
    "                                [dWxh, dWhh, dWhy, dbh, dby], \n",
    "                                [mWxh, mWhh, mWhy, mbh, mby]):\n",
    "    mem += dparam * dparam\n",
    "    param += -learning_rate * dparam / np.sqrt(mem + 1e-8) # adagrad update\n",
    "\n",
    "  p += seq_length # move data pointer\n",
    "  n += 1 # iteration counter "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pygame 1.9.5\n",
      "Hello from the pygame community. https://www.pygame.org/contribute.html\n"
     ]
    }
   ],
   "source": [
    "# play the pice in the list (copy-paste from RNN output)\n",
    "import pygame.midi\n",
    "import time\n",
    "import random\n",
    "\n",
    "pygame.midi.init()\n",
    "player = pygame.midi.Output(0)\n",
    "player.set_instrument(0) # https://pjb.com.au/muscript/gm.html\n",
    "# channel - note - velocity - time\n",
    "notel = ['1:75:80', '0:70:0', '2:51:0', '0:77:80', '2:50:80', '1:67:0', '2:51:0', '0:74:80', '1:60:0', '1:72:80', '2:53:0', '0:70:0', '0:72:0', '0:67:80', '0:70:0', '0:70:80', '1:70:0', '2:50:0', '1:72:80', '0:70:0', '1:70:0', '0:70:0', '1:68:0', '0:72:80', '2:55:80', '2:51:0', '0:77:80', '0:77:0', '0:70:0', '0:72:80', '2:51:0', '1:67:0', '2:48:80', '1:60:0', '0:78:80', '0:70:0', '0:79:80', '0:70:0', '0:67:80', '2:51:0', '0:77:80', '2:50:0', '0:70:80', '0:70:0', '0:67:0', '0:75:80', '0:70:0', '0:72:80', '1:72:0', '0:79:80', '0:72:0', '0:72:80', '1:70:0', '1:72:0', '1:72:80', '0:74:0', '1:60:0', '0:70:0', '0:72:80', '1:72:0', '1:72:80', '2:53:80', '0:67:0', '0:65:80', '0:67:0', '1:74:80', '0:72:80', '0:70:0', '0:70:80', '0:72:0', '1:63:80', '1:68:0', '0:72:80', '0:74:0', '1:70:0', '0:70:80', '2:58:0', '0:72:0', '0:70:80', '2:53:80', '1:72:0', '1:70:80', '2:48:80', '1:70:0', '1:70:80', '1:72:0', '1:70:0', '1:70:80', '1:70:0', '1:72:80', '1:70:0', '1:67:80', '1:72:0', '1:72:80', '1:72:0', '1:70:80', '1:72:0', '1:70:80', '0:74:0', '0:75:0', '0:72:80', '0:72:0', '0:72:0', '1:63:0', '0:72:80', '1:63:0', '2:51:80', '0:72:0', '0:72:80', '1:70:0', '1:70:80', '2:50:80', '0:60:0', '0:67:0', '1:70:80', '1:72:0', '2:46:80', '2:50:80', '0:67:0', '1:77:0', '0:70:80', '2:51:80', '0:70:0', '1:67:0', '0:65:80', '1:70:0', '0:64:80', '2:50:0', '0:72:80', '0:70:0', '2:50:0', '0:72:80', '2:50:0', '1:70:0', '0:70:80', '2:50:80', '0:70:0', '1:67:0', '1:77:0', '0:72:80', '0:70:0', '0:72:80', '1:72:0', '1:70:80']\n",
    "for n in notel:\n",
    "    ns = n.split(':')\n",
    "    #print ns\n",
    "    player.note_on(int(ns[1]), int(ns[2]), int(ns[0])) # note_on(note, velocity=None, channel = 0) https://en.scratch-wiki.info/wiki/MIDI_Notes\n",
    "    time.sleep(int(ns[2])/200)\n",
    "    player.note_off(int(ns[1]), int(ns[2]), int(ns[0]))\n",
    "del player\n",
    "pygame.midi.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['2:51:80', '1:63:80', '1:63:0', '1:67:80', '1:67:0', '1:70:80', '2:51:0', '1:70:0', '1:75:80', '2:55:80', '1:75:0', '1:70:80', '2:55:0', '1:70:0', '1:72:80', '2:56:80', '1:72:0', '0:80:80', '1:84:80', '2:56:0', '0:80:0', '1:84:0', '0:79:80', '1:82:80', '2:58:80', '0:79:0', '1:82:0', '0:77:80', '1:80:80', '2:58:0', '0:77:0', '1:80:0', '0:75:80', '1:79:80', '2:46:80', '0:75:0', '1:79:0', '0:74:80', '1:77:80', '2:46:0', '0:74:0', '1:77:0', '0:75:80', '1:79:80', '2:51:80', '1:79:0', '1:63:80', '0:75:0', '1:63:0', '0:63:80', '1:67:80', '1:67:0', '1:70:80', '2:51:0', '0:63:0', '1:70:0', '0:63:80', '1:75:80', '2:55:80', '1:75:0', '1:70:80', '0:63:0', '2:55:0', '1:70:0', '0:63:80', '1:72:80', '2:56:80', '0:63:0', '1:72:0', '0:80:80', '1:84:80', '2:56:0', '0:80:0', '1:84:0', '0:79:80', '1:82:80', '2:58:80', '0:79:0', '1:82:0', '0:77:80', '1:80:80', '2:58:0', '0:77:0', '1:80:0', '0:75:80', '1:79:80', '2:46:80', '0:75:0', '1:79:0', '0:74:80', '1:77:80', '2:46:0', '0:74:0', '1:77:0', '0:75:80', '1:79:80', '2:51:80', '1:79:0', '1:82:80', '0:75:0', '1:82:0', '0:82:80', '1:80:80', '1:80:0', '1:79:80', '2:51:0', '0:82:0', '1:79:0', '0:84:80', '1:77:80', '2:56:80', '1:77:0', '1:75:80', '0:84:0', '2:56:0', '1:75:0', '0:77:80', '1:74:80', '2:58:80', '0:77:0', '1:74:0', '2:58:0', '0:79:80', '1:75:80', '2:63:80', '2:63:0', '2:58:80', '2:58:0', '2:55:80', '0:79:0', '1:75:0', '2:55:0', '0:80:80', '1:77:80', '2:51:80', '0:80:0', '1:77:0', '0:82:80', '1:79:80', '2:51:0', '0:82:0', '1:79:0', '0:77:80', '1:74:80', '2:58:80', '0:77:0', '1:74:0', '2:58:0', '0:79:80', '1:75:80', '2:63:80', '2:63:0', '2:58:80', '2:58:0', '2:55:80', '0:79:0', '1:75:0', '2:55:0', '0:80:80', '1:77:80', '2:51:80', '0:80:0', '1:77:0', '0:82:80', '1:79:80', '2:51:0', '0:82:0', '1:79:0', '0:77:80', '1:74:80', '2:58:80', '0:77:0', '1:74:0', '0:75:80', '1:72:80', '2:58:0', '0:75:0', '1:72:0', '0:74:80', '1:70:80', '2:46:80', '0:74:0', '1:70:0', '0:72:80', '1:68:80', '2:46:0', '0:72:0', '1:68:0', '0:70:80', '1:67:80', '2:58:80', '0:70:0', '1:67:0', '0:68:80', '1:65:80', '2:58:0', '0:68:0', '1:65:0', '0:67:80', '1:63:80', '2:51:80', '0:67:0', '0:63:80', '0:63:0', '0:67:80', '0:67:0', '0:70:80', '1:63:0', '2:51:0', '0:70:0', '0:75:80', '1:70:80', '2:55:80', '0:75:0', '1:70:0', '0:70:80', '1:67:80', '2:55:0', '0:70:0', '1:67:0', '0:72:80', '1:63:80', '2:56:80', '0:72:0', '1:63:0', '0:84:80', '1:80:80', '2:56:0', '0:84:0', '1:80:0', '0:82:80', '1:79:80', '2:58:80', '0:82:0', '1:79:0', '0:80:80', '1:77:80', '2:58:0', '0:80:0', '1:77:0', '0:79:80', '1:75:80', '2:46:80', '0:79:0', '1:75:0', '0:77:80', '1:74:80', '2:46:0', '0:77:0', '1:74:0', '0:79:80', '1:75:80', '2:51:80', '0:79:0', '0:63:80', '1:75:0', '0:63:0', '0:67:80', '1:63:80', '0:67:0', '0:70:80', '2:51:0', '1:63:0', '0:70:0', '0:75:80', '1:63:80', '2:55:80', '0:75:0', '0:70:80', '1:63:0', '2:55:0', '0:70:0', '0:72:80', '1:63:80', '2:56:80', '0:72:0', '1:63:0', '0:84:80', '1:80:80', '2:56:0', '0:84:0', '1:80:0', '0:82:80', '1:79:80', '2:58:80', '0:82:0', '1:79:0', '0:80:80', '1:77:80', '2:58:0', '0:80:0', '1:77:0', '0:79:80', '1:75:80', '2:46:80', '0:79:0', '1:75:0', '0:77:80', '1:74:80', '2:46:0', '0:77:0', '1:74:0', '0:79:80', '1:75:80', '2:51:80', '0:79:0', '0:77:80', '1:75:0', '2:51:0', '0:77:0', '0:75:80', '1:79:80', '2:51:80', '0:75:0', '0:77:80', '1:79:0', '2:51:0', '0:77:0', '0:79:80', '1:75:80', '2:51:80', '0:79:0', '0:80:80', '1:75:0', '2:51:0', '0:80:0', '0:82:80', '1:77:80', '2:50:80', '0:82:0', '0:77:80', '1:77:0', '2:50:0', '0:77:0', '0:74:80', '1:82:80', '2:50:80', '0:74:0', '0:70:80', '1:82:0', '2:50:0', '0:70:0', '0:77:80', '2:50:80', '0:77:0', '0:74:80', '2:50:0', '0:74:0', '0:75:80', '2:48:80', '0:75:0', '0:74:80', '2:48:0', '0:74:0', '0:72:80', '1:75:80', '2:48:80', '0:72:0', '0:74:80', '1:75:0', '2:48:0', '0:74:0', '0:75:80', '1:72:80', '2:48:80', '0:75:0', '0:77:80', '1:72:0', '2:48:0', '0:77:0', '0:79:80', '1:74:80', '2:46:80', '0:79:0', '0:74:80', '1:74:0', '2:46:0', '0:74:0', '0:70:80', '1:79:80', '2:46:80', '0:70:0', '0:67:80', '1:79:0', '2:46:0', '0:67:0', '0:74:80', '2:46:80', '0:74:0', '0:70:80', '2:46:0', '0:70:0', '0:72:80', '2:44:80', '0:72:0', '0:68:80', '2:44:0', '0:68:0', '0:75:80', '1:72:80', '2:56:80', '0:75:0', '0:72:80', '1:72:0', '2:56:0', '0:72:0', '0:70:80', '1:75:80', '2:56:80', '0:70:0', '0:68:80', '2:56:0', '0:68:0', '0:70:80', '2:55:80', '0:70:0', '0:67:80', '1:75:0', '2:55:0', '0:67:0', '0:75:80', '1:70:80', '2:55:80', '0:75:0', '0:70:80', '1:70:0', '2:55:0', '0:70:0', '0:68:80', '1:75:80', '2:55:80', '0:68:0', '0:67:80', '1:75:0', '2:55:0', '0:67:0', '0:68:80', '1:74:80', '2:53:80', '0:68:0', '1:74:0', '0:67:80', '1:75:80', '2:53:0', '0:67:0', '1:75:0', '0:68:80', '1:77:80', '2:50:80', '0:68:0', '0:70:80', '1:77:0', '2:50:0', '0:70:0', '0:67:80', '1:75:80', '2:51:80', '0:67:0', '0:68:80', '1:75:0', '2:51:0', '0:68:0', '0:65:80', '1:74:80', '2:46:80', '0:65:0', '0:72:80', '1:74:0', '2:46:0', '0:72:0', '0:70:80', '2:50:80', '0:70:0', '0:68:80', '2:50:0', '0:68:0', '0:67:80', '2:46:80', '0:67:0', '0:65:80', '2:46:0', '0:65:0', '0:67:80', '2:51:80', '0:67:0', '0:77:80', '2:51:0', '0:77:0', '0:75:80', '2:48:80', '0:75:0', '0:74:80', '2:48:0', '0:74:0', '0:72:80', '2:51:80', '0:72:0', '0:70:80', '2:51:0', '0:70:0', '0:69:80', '2:53:80', '0:69:0', '0:79:80', '2:53:0', '0:79:0', '0:77:80', '2:57:80', '0:77:0', '0:75:80', '2:57:0', '0:75:0', '0:74:80', '2:53:80', '0:74:0', '0:72:80', '2:53:0', '0:72:0', '0:74:80', '2:58:80', '1:65:80', '0:74:0', '2:58:0', '1:65:0', '0:74:80', '1:70:80', '2:58:80', '1:70:0', '1:72:80', '0:74:0', '2:58:0', '1:72:0', '0:70:80', '1:74:80', '2:58:80', '1:74:0', '1:75:80', '0:70:0', '2:58:0', '1:75:0', '0:72:80', '1:77:80', '2:57:80', '1:77:0', '1:72:80', '0:72:0', '2:57:0', '1:72:0', '0:77:80', '1:69:80', '2:57:80', '1:69:0', '1:65:80', '0:77:0', '2:57:0', '1:65:0', '1:72:80', '2:57:80', '1:72:0', '1:69:80', '2:57:0', '1:69:0', '1:70:80', '2:55:80', '1:70:0', '1:69:80', '2:55:0', '1:69:0', '0:70:80', '1:67:80', '2:55:80', '1:67:0', '1:69:80', '0:70:0', '2:55:0', '1:69:0', '0:67:80', '1:70:80', '2:55:80', '1:70:0', '1:72:80', '0:67:0', '2:55:0', '1:72:0', '0:69:80', '1:74:80', '2:53:80', '1:74:0', '1:69:80', '0:69:0', '2:53:0', '1:69:0', '0:74:80', '1:65:80', '2:53:80', '1:65:0', '1:62:80', '0:74:0', '2:53:0', '1:62:0', '1:69:80', '2:53:80', '1:69:0', '1:65:80', '2:53:0', '1:65:0', '1:67:80', '2:51:80', '1:67:0', '1:63:80', '2:51:0', '1:63:0', '0:67:80', '1:70:80', '2:51:80', '1:70:0', '1:67:80', '0:67:0', '2:51:0', '1:67:0', '0:70:80', '1:65:80', '2:51:80', '1:65:0', '1:63:80', '2:51:0', '1:63:0', '1:65:80', '2:50:80', '1:65:0', '1:62:80', '0:70:0', '2:50:0', '1:62:0', '0:65:80', '1:70:80', '2:50:80', '1:70:0', '1:65:80', '0:65:0', '2:50:0', '1:65:0', '0:70:80', '1:63:80', '2:50:80', '1:63:0', '1:62:80', '0:70:0', '2:50:0', '1:62:0', '0:69:80', '1:63:80', '2:48:80', '0:69:0', '1:63:0', '0:70:80', '1:62:80', '2:48:0', '0:70:0', '1:62:0', '0:72:80', '1:63:80', '2:45:80', '1:63:0', '1:65:80', '0:72:0', '2:45:0', '1:65:0', '0:70:80', '1:62:80', '2:46:80', '1:62:0', '1:63:80', '0:70:0', '2:46:0', '1:63:0', '0:79:80', '1:60:80', '2:51:80', '0:79:0', '0:80:80', '1:60:0', '2:51:0', '0:80:0', '0:79:80', '1:72:80', '2:51:80', '0:79:0', '0:77:80', '1:72:0', '2:51:0', '0:77:0', '0:75:80', '2:51:80', '0:75:0', '0:74:80', '2:51:0', '0:74:0', '0:75:80', '1:79:80', '2:51:80', '0:75:0', '1:79:0', '0:77:80', '1:80:80', '2:51:0', '0:77:0', '1:80:0', '0:75:80', '1:79:80', '2:51:80', '0:75:0', '1:79:0', '0:74:80', '1:77:80', '2:51:0', '0:74:0', '1:77:0', '0:72:80', '1:75:80', '2:51:80', '0:72:0', '1:75:0', '0:70:80', '1:74:80', '2:51:0', '0:70:0', '1:74:0', '0:69:80', '1:72:80', '2:53:80', '1:72:0', '1:77:80', '0:69:0', '1:77:0', '0:77:80', '1:81:80', '1:81:0', '1:72:80', '2:53:0', '0:77:0', '1:72:0', '0:77:80', '1:74:80', '2:58:80', '1:74:0', '1:82:80', '0:77:0', '2:58:0', '1:82:0', '0:77:80', '1:75:80', '2:57:80', '0:77:0', '0:79:80', '1:75:0', '0:79:0', '0:77:80', '1:74:80', '0:77:0', '0:79:80', '2:57:0', '1:74:0', '0:79:0', '0:77:80', '1:75:80', '2:53:80', '0:77:0', '0:79:80', '1:75:0', '0:79:0', '0:77:80', '1:84:80', '0:77:0', '0:79:80', '2:53:0', '1:84:0', '0:79:0', '0:77:80', '1:74:80', '2:58:80', '0:77:0', '0:79:80', '1:74:0', '0:79:0', '0:77:80', '1:82:80', '0:77:0', '0:79:80', '2:58:0', '1:82:0', '0:79:0', '0:77:80', '1:72:80', '2:53:80', '0:77:0', '0:79:80', '1:72:0', '0:79:0', '0:77:80', '1:77:80', '0:77:0', '0:79:80', '1:77:0', '0:79:0', '0:77:80', '1:81:80', '0:77:0', '0:79:80', '1:81:0', '0:79:0', '0:77:80', '1:72:80', '2:53:0', '0:77:0', '0:79:80', '1:72:0', '0:79:0', '0:77:80', '1:74:80', '2:58:80', '0:77:0', '0:79:80', '1:74:0', '0:79:0', '0:77:80', '1:82:80', '0:77:0', '0:79:80', '2:58:0', '1:82:0', '0:79:0', '0:77:80', '1:75:80', '2:57:80', '0:77:0', '0:79:80', '1:75:0', '0:79:0', '0:77:80', '1:74:80', '0:77:0', '0:79:80', '2:57:0', '1:74:0', '0:79:0', '0:77:80', '1:75:80', '2:53:80', '0:77:0', '0:79:80', '1:75:0', '0:79:0', '0:77:80', '1:84:80', '0:77:0', '0:79:80', '2:53:0', '1:84:0', '0:79:0', '0:77:80', '1:74:80', '2:58:80', '0:77:0', '0:79:80', '1:74:0', '0:79:0', '0:77:80', '1:82:80', '0:77:0', '0:79:80', '2:58:0', '1:82:0', '0:79:0', '0:77:80', '1:72:80', '2:53:80', '1:72:0', '1:77:80', '0:77:0', '2:53:0', '1:77:0', '0:72:80', '1:81:80', '2:41:80', '0:72:0', '1:81:0', '0:70:80', '1:79:80', '2:41:0', '0:70:0', '1:79:0', '0:72:80', '1:81:80', '2:41:80', '0:72:0', '1:81:0', '0:74:80', '1:82:80', '2:41:0', '0:74:0', '1:82:0', '0:75:80', '1:84:80', '2:41:80', '0:75:0', '1:84:0', '0:74:80', '1:82:80', '2:41:0', '0:74:0', '1:82:0', '0:72:80', '1:81:80', '2:41:80', '0:72:0', '1:81:0', '0:75:80', '1:79:80', '2:41:0', '0:75:0', '1:79:0', '0:74:80', '1:77:80', '2:41:80', '0:74:0', '1:77:0', '0:72:80', '1:75:80', '2:41:0', '0:72:0', '1:75:0', '1:74:80', '2:46:80', '0:70:80', '1:74:0', '0:70:0', '0:74:80', '1:70:80', '0:74:0', '0:65:80', '2:46:0', '1:70:0', '0:65:0', '0:67:80', '1:70:80', '2:51:80', '0:67:0', '0:75:80', '1:70:0', '2:51:0', '0:75:0', '0:68:80', '1:70:80', '2:50:80', '1:70:0', '1:72:80', '0:68:0', '0:67:80', '1:72:0', '1:70:80', '1:70:0', '1:72:80', '2:50:0', '0:67:0', '0:68:80', '1:72:0', '1:70:80', '2:46:80', '1:70:0', '1:72:80', '0:68:0', '0:77:80', '1:72:0', '1:70:80', '1:70:0', '1:72:80', '2:46:0', '0:77:0', '0:67:80', '1:72:0', '1:70:80', '2:51:80', '1:70:0', '1:72:80', '0:67:0', '0:75:80', '1:72:0', '1:70:80', '1:70:0', '1:72:80', '2:51:0', '0:75:0', '0:65:80', '1:72:0', '1:70:80', '2:46:80', '1:70:0', '1:72:80', '0:65:0', '0:70:80', '1:72:0', '1:70:80', '1:70:0', '1:72:80', '0:70:0', '0:74:80', '1:72:0', '1:70:80', '1:70:0', '1:72:80', '0:74:0', '0:65:80', '1:72:0', '1:70:80', '2:46:0', '1:70:0', '1:72:80', '0:65:0', '0:67:80', '1:72:0', '1:70:80', '2:51:80', '1:70:0', '1:72:80', '0:67:0', '0:75:80', '1:72:0', '1:70:80', '1:70:0', '1:72:80', '2:51:0', '0:75:0', '0:68:80', '1:72:0', '1:70:80', '2:50:80', '1:70:0', '1:72:80', '0:68:0', '0:67:80', '1:72:0', '1:70:80', '1:70:0', '1:72:80', '2:50:0', '0:67:0', '0:68:80', '1:72:0', '1:70:80', '2:46:80', '1:70:0', '1:72:80', '0:68:0', '0:77:80', '1:72:0', '1:70:80', '1:70:0', '1:72:80', '2:46:0', '0:77:0', '0:67:80', '1:72:0', '1:70:80', '2:51:80', '1:70:0', '1:72:80', '0:67:0', '0:75:80', '1:72:0', '1:70:80', '1:70:0', '1:72:80', '2:51:0', '0:75:0', '0:65:80', '1:72:0', '1:70:80', '2:46:80', '0:65:0', '0:70:80', '1:70:0', '2:46:0', '0:70:0', '0:74:80', '1:70:80', '2:46:80', '0:74:0', '1:70:0', '0:75:80', '1:72:80', '2:46:0', '0:75:0', '1:72:0', '0:77:80', '1:74:80', '2:46:80', '0:77:0', '1:74:0', '0:79:80', '1:75:80', '2:46:0', '0:79:0', '1:75:0', '0:80:80', '1:77:80', '2:46:80', '0:80:0', '1:77:0', '0:79:80', '1:75:80', '2:46:0', '0:79:0', '1:75:0', '0:77:80', '1:74:80', '2:58:80', '0:77:0', '1:74:0', '0:80:80', '1:77:80', '2:58:0', '0:80:0', '1:77:0', '0:79:80', '1:75:80', '2:46:80', '0:79:0', '1:75:0', '0:77:80', '1:74:80', '2:46:0', '0:77:0', '1:74:0', '0:79:80', '1:75:80', '2:51:80', '0:79:0', '0:77:80', '1:75:0', '2:51:0', '0:77:0', '0:75:80', '1:79:80', '2:48:80', '0:75:0', '1:79:0', '0:74:80', '1:77:80', '2:48:0', '0:74:0', '1:77:0', '0:72:80', '1:75:80', '2:51:80', '0:72:0', '1:75:0', '0:70:80', '1:74:80', '2:51:0', '0:70:0', '1:74:0', '0:69:80', '1:72:80', '2:53:80', '0:69:0', '1:72:0', '0:70:80', '1:70:80', '2:53:0', '0:70:0', '1:70:0', '0:72:80', '1:69:80', '2:41:80', '0:72:0', '1:69:0', '0:75:80', '1:72:80', '2:41:0', '0:75:0', '1:72:0', '0:74:80', '1:70:80', '2:53:80', '0:74:0', '1:70:0', '0:72:80', '1:69:80', '2:53:0', '0:72:0', '1:69:0', '0:74:80', '1:70:80', '2:58:80', '0:74:0', '1:70:0', '0:75:80', '1:72:80', '2:58:0', '0:75:0', '1:72:0', '0:77:80', '1:74:80', '2:46:80', '2:46:0', '2:48:80', '2:48:0', '2:50:80', '2:50:0', '2:51:80', '2:51:0', '2:53:80', '0:77:0', '1:74:0', '0:72:80', '1:69:80', '2:53:0', '0:72:0', '1:69:0', '0:72:80', '1:69:80', '2:41:80', '0:72:0', '1:69:0', '0:75:80', '1:72:80', '2:41:0', '0:75:0', '1:72:0', '0:74:80', '1:70:80', '2:53:80', '0:74:0', '1:70:0', '0:72:80', '1:69:80', '2:53:0', '0:72:0', '1:69:0', '0:74:80', '1:70:80', '2:46:80', '0:74:0', '1:70:0', '0:75:80', '1:72:80', '2:46:0', '0:75:0', '1:72:0', '0:77:80', '1:74:80', '2:46:80', '2:46:0', '2:48:80', '2:48:0', '2:50:80', '2:50:0', '2:51:80', '2:51:0', '2:53:80', '0:77:0', '1:74:0', '0:72:80', '1:69:80', '2:53:0', '0:72:0', '1:69:0', '0:72:80', '1:69:80', '2:41:80', '0:72:0', '1:69:0', '0:75:80', '1:72:80', '2:41:0', '0:75:0', '1:72:0', '0:74:80', '1:70:80', '2:53:80', '0:74:0', '1:70:0', '0:72:80', '1:69:80', '2:53:0', '0:72:0', '1:69:0', '0:74:80', '1:70:80', '2:46:80', '0:74:0', '0:75:80', '2:46:0', '0:75:0', '0:75:80', '2:48:80', '0:75:0', '0:77:80', '2:48:0', '0:77:0', '0:77:80', '2:50:80', '0:77:0', '0:79:80', '2:50:0', '0:79:0', '0:79:80', '2:51:80', '0:79:0', '0:81:80', '1:70:0', '1:75:80', '2:51:0', '0:81:0', '1:75:0', '0:81:80', '1:75:80', '2:53:80', '0:81:0', '0:82:80', '1:75:0', '2:53:0', '0:82:0', '0:82:80', '1:74:80', '2:55:80', '0:82:0', '0:70:80', '1:74:0', '2:55:0', '0:70:0', '0:79:80', '1:72:80', '2:51:80', '0:79:0', '0:75:80', '1:72:0', '2:51:0', '0:75:0', '0:74:80', '1:70:80', '2:53:80', '0:74:0', '1:70:0', '2:53:0', '0:72:80', '1:69:80', '2:41:80', '0:72:0', '1:69:0', '2:41:0', '0:70:80', '1:74:80', '2:46:80', '0:70:0', '1:74:0', '0:72:80', '1:75:80', '2:46:0', '0:72:0', '1:75:0', '0:74:80', '1:77:80', '2:46:80', '2:46:0', '2:48:80', '2:48:0', '2:50:80', '2:50:0', '2:51:80', '2:51:0', '2:53:80', '0:74:0', '1:77:0', '0:69:80', '1:72:80', '2:53:0', '0:69:0', '1:72:0', '0:69:80', '1:72:80', '2:41:80', '0:69:0', '1:72:0', '0:72:80', '1:75:80', '2:41:0', '0:72:0', '1:75:0', '0:70:80', '1:74:80', '2:53:80', '0:70:0', '1:74:0', '0:69:80', '1:72:80', '2:53:0', '0:69:0', '1:72:0', '0:70:80', '1:74:80', '2:46:80', '0:70:0', '1:74:0', '0:72:80', '1:75:80', '2:46:0', '0:72:0', '1:75:0', '0:74:80', '1:77:80', '2:46:80', '2:46:0', '2:48:80', '2:48:0', '2:50:80', '2:50:0', '2:51:80', '2:51:0', '2:53:80', '0:74:0', '1:77:0', '0:69:80', '1:72:80', '2:53:0', '0:69:0', '1:72:0', '0:69:80', '1:72:80', '2:41:80', '0:69:0', '1:72:0', '0:72:80', '1:75:80', '2:41:0', '0:72:0', '1:75:0', '0:70:80', '1:74:80', '2:53:80', '0:70:0', '1:74:0', '0:69:80', '1:72:80', '2:53:0', '0:69:0', '1:72:0', '0:70:80', '1:74:80', '2:46:80', '0:70:0', '1:74:0', '0:74:80', '1:77:80', '2:46:0', '0:74:0', '1:77:0', '0:77:80', '1:82:80', '2:50:80', '1:82:0', '1:81:80', '2:50:0', '1:81:0', '1:79:80', '2:46:80', '1:79:0', '1:77:80', '2:46:0', '1:77:0', '1:79:80', '2:51:80', '1:79:0', '1:77:80', '0:77:0', '1:77:0', '0:75:80', '1:79:80', '2:51:0', '1:79:0', '1:81:80', '2:53:80', '0:75:0', '1:81:0', '2:53:0', '0:74:80', '1:82:80', '2:55:80', '0:74:0', '1:82:0', '2:55:0', '0:75:80', '1:79:80', '2:51:80', '0:75:0', '1:79:0', '0:72:80', '1:75:80', '2:51:0', '0:72:0', '1:75:0', '0:70:80', '1:74:80', '2:53:80', '0:70:0', '1:74:0', '0:69:80', '1:72:80', '0:69:0', '1:72:0', '0:70:80', '1:74:80', '0:70:0', '1:74:0', '0:69:80', '1:72:80', '2:53:0', '0:69:0', '1:72:0', '0:70:80', '1:74:80', '0:70:0', '1:74:0', '0:69:80', '1:72:80', '2:41:80', '0:69:0', '1:72:0', '0:70:80', '1:70:80', '2:41:0', '0:70:0', '1:70:0', '0:70:80', '1:70:80', '2:46:80', '0:70:0', '1:70:0', '2:46:0', '2:46:80', '1:58:80', '1:58:0', '1:62:80', '1:62:0', '1:65:80', '2:46:0', '1:65:0', '1:70:80', '2:50:80', '1:70:0', '1:65:80', '2:50:0', '1:65:0', '1:67:80', '2:51:80', '1:67:0', '0:75:80', '1:79:80', '2:51:0', '0:75:0', '1:79:0', '0:74:80', '1:77:80', '2:53:80', '0:74:0', '1:77:0', '0:72:80', '1:75:80', '2:53:0', '0:72:0', '1:75:0', '0:70:80', '1:74:80', '2:41:80', '0:70:0', '1:74:0', '0:69:80', '1:72:80', '2:41:0', '0:69:0', '1:72:0', '0:70:80', '1:74:80', '2:46:80', '1:74:0', '1:58:80', '0:70:0', '1:58:0', '0:82:80', '1:62:80', '1:62:0', '1:65:80', '2:46:0', '0:82:0', '1:65:0', '0:70:80', '1:70:80', '2:50:80', '1:70:0', '1:65:80', '2:50:0', '1:65:0', '1:67:80', '2:51:80', '0:70:0', '1:67:0', '0:75:80', '1:79:80', '2:51:0', '0:75:0', '1:79:0', '0:74:80', '1:77:80', '2:53:80', '0:74:0', '1:77:0', '0:72:80', '1:75:80', '2:53:0', '0:72:0', '1:75:0', '0:70:80', '1:74:80', '2:41:80', '0:70:0', '1:74:0', '0:69:80', '1:72:80', '2:41:0', '0:69:0', '1:72:0', '0:70:80', '1:74:80', '2:46:80', '1:74:0', '1:77:80', '0:70:0', '1:77:0', '0:77:80', '1:75:80', '1:75:0', '1:74:80', '2:46:0', '0:77:0', '1:74:0', '0:79:80', '1:72:80', '2:51:80', '1:72:0', '1:70:80', '0:79:0', '2:51:0', '1:70:0', '0:72:80', '1:69:80', '2:53:80', '0:72:0', '1:69:0', '2:53:0', '0:74:80', '1:70:80', '2:58:80', '2:58:0', '2:53:80', '2:53:0', '2:50:80', '0:74:0', '1:70:0', '2:50:0', '0:75:80', '1:72:80', '2:46:80', '0:75:0', '1:72:0', '0:77:80', '1:74:80', '2:46:0', '0:77:0', '1:74:0', '0:72:80', '1:69:80', '2:53:80', '0:72:0', '1:69:0', '2:53:0', '0:74:80', '1:70:80', '2:58:80', '2:58:0', '2:53:80', '2:53:0', '2:50:80', '0:74:0', '1:70:0', '2:50:0', '0:75:80', '1:72:80', '2:46:80', '0:75:0', '1:72:0', '0:77:80', '1:74:80', '2:46:0', '0:77:0', '1:74:0', '0:72:80', '1:69:80', '2:53:80', '0:72:0', '0:65:80', '2:53:0', '2:51:80', '1:69:0', '0:65:0', '2:51:0', '0:77:80', '2:50:80', '2:50:0', '2:48:80', '2:48:0', '2:47:80', '0:77:0', '0:80:80', '0:80:0', '0:79:80', '1:74:80', '0:79:0', '0:77:80', '0:77:0', '0:75:80', '0:75:0', '2:47:0', '0:74:80', '1:74:0', '0:74:0', '0:75:80', '1:74:80', '2:48:80', '0:75:0', '2:48:0', '0:74:80', '2:47:80', '1:74:0', '0:74:0', '2:47:0', '0:75:80', '1:67:80', '2:48:80', '0:75:0', '2:48:0', '0:77:80', '2:50:80', '1:67:0', '0:77:0', '2:50:0', '0:79:80', '1:72:80', '2:51:80', '1:72:0', '2:51:0', '1:70:80', '2:48:80', '1:70:0', '2:48:0', '1:68:80', '2:53:80', '0:79:0', '1:68:0', '0:77:80', '1:80:80', '0:77:0', '1:80:0', '0:75:80', '1:79:80', '0:75:0', '1:79:0', '0:74:80', '1:77:80', '0:74:0', '1:77:0', '2:53:0', '0:72:80', '1:75:80', '0:72:0', '1:75:0', '0:71:80', '1:74:80', '2:55:80', '0:71:0', '1:74:0', '2:55:0', '0:72:80', '1:75:80', '2:60:80', '2:60:0', '2:55:80', '2:55:0', '2:51:80', '0:72:0', '1:75:0', '2:51:0', '0:74:80', '1:77:80', '2:48:80', '0:74:0', '1:77:0', '0:75:80', '1:79:80', '2:48:0', '0:75:0', '1:79:0', '0:71:80', '1:74:80', '2:55:80', '0:71:0', '1:74:0', '2:55:0', '0:72:80', '1:75:80', '2:60:80', '2:60:0', '2:55:80', '2:55:0', '2:51:80', '0:72:0', '1:75:0', '2:51:0', '0:74:80', '1:77:80', '2:48:80', '0:74:0', '1:77:0', '0:75:80', '1:79:80', '2:48:0', '0:75:0', '1:79:0', '0:71:80', '1:74:80', '2:55:80', '0:71:0', '0:79:80', '1:74:0', '2:55:0', '0:79:0', '0:79:80', '2:59:80', '0:79:0', '0:77:80', '2:59:0', '0:77:0', '0:77:80', '2:55:80', '0:77:0', '0:75:80', '2:55:0', '0:75:0', '0:74:80', '2:60:80', '1:84:80', '0:74:0', '2:60:0', '1:84:0', '0:76:80', '1:84:80', '2:48:80', '1:84:0', '1:82:80', '2:48:0', '1:82:0', '1:82:80', '2:60:80', '1:82:0', '1:80:80', '0:76:0', '2:60:0', '1:80:0', '1:80:80', '2:53:80', '0:77:80', '1:80:0', '2:53:0', '0:77:0', '0:77:80', '1:81:80', '2:41:80', '0:77:0', '0:75:80', '2:41:0', '0:75:0', '0:75:80', '2:53:80', '0:75:0', '0:74:80', '1:81:0', '2:53:0', '0:74:0', '0:74:80', '2:58:80', '1:82:80', '0:74:0', '2:58:0', '1:82:0', '0:74:80', '1:82:80', '2:46:80', '1:82:0', '1:80:80', '2:46:0', '1:80:0', '1:80:80', '2:58:80', '1:80:0', '1:79:80', '0:74:0', '2:58:0', '1:79:0', '0:75:80', '1:79:80', '2:51:80', '1:79:0', '1:79:80', '0:75:0', '2:51:0', '1:79:0', '0:63:80', '1:80:80', '2:48:80', '1:80:0', '1:79:80', '0:63:0', '2:48:0', '1:79:0', '0:75:80', '1:77:80', '2:60:80', '1:77:0', '1:75:80', '0:75:0', '1:75:0', '0:74:80', '1:77:80', '1:77:0', '1:77:80', '0:74:0', '2:60:0', '1:77:0', '0:75:80', '1:79:80', '2:59:80', '0:75:0', '1:79:0', '0:74:80', '1:77:80', '0:74:0', '0:75:80', '0:75:0', '0:74:80', '0:74:0', '0:75:80', '1:77:0', '0:75:0', '0:74:80', '1:75:80', '1:75:0', '1:74:80', '2:59:0', '0:74:0', '1:74:0', '1:75:80', '2:60:80', '0:79:80', '1:75:0', '2:60:0', '0:79:0', '0:80:80', '1:63:80', '2:48:80', '0:80:0', '0:79:80', '1:63:0', '2:48:0', '0:79:0', '0:77:80', '1:75:80', '2:60:80', '0:77:0', '0:75:80', '1:75:0', '2:60:0', '0:75:0', '0:77:80', '1:74:80', '2:60:80', '0:77:0', '0:77:80', '1:74:0', '2:60:0', '0:77:0', '0:79:80', '1:74:80', '2:59:80', '0:79:0', '0:77:80', '0:77:0', '0:75:80', '0:75:0', '0:74:80', '1:74:0', '2:59:0', '0:74:0', '0:75:80', '2:60:80', '0:75:0', '1:60:80', '2:60:0', '2:58:80', '1:60:0', '2:58:0', '1:72:80', '2:57:80', '2:57:0', '2:55:80', '2:55:0', '2:54:80', '1:72:0', '1:75:80', '1:75:0', '0:69:80', '1:74:80', '1:74:0', '1:72:80', '1:72:0', '1:70:80', '1:70:0', '2:54:0', '1:69:80', '1:69:0', '1:70:80', '2:55:80', '1:70:0', '2:55:0', '1:69:80', '2:54:80', '0:69:0', '1:69:0', '2:54:0', '0:62:80', '1:70:80', '2:55:80', '1:70:0', '2:55:0', '1:72:80', '2:57:80', '0:62:0', '1:72:0', '2:57:0', '0:67:80', '1:74:80', '2:59:80', '0:67:0', '2:59:0', '0:65:80', '2:55:80', '0:65:0', '2:55:0', '0:63:80', '2:60:80', '0:63:0', '1:74:0', '2:60:0', '0:75:80', '1:72:80', '2:48:80', '0:75:0', '1:72:0', '0:74:80', '1:70:80', '0:74:0', '1:70:0', '0:72:80', '1:69:80', '0:72:0', '1:69:0', '0:70:80', '1:67:80', '2:48:0', '0:70:0', '1:67:0', '0:69:80', '1:66:80', '2:50:80', '0:69:0', '1:66:0', '2:50:0', '0:70:80', '1:67:80', '2:55:80', '2:55:0', '2:50:80', '2:50:0', '2:46:80', '0:70:0', '1:67:0', '2:46:0', '0:72:80', '1:69:80', '2:43:80', '0:72:0', '1:69:0', '0:74:80', '1:70:80', '2:43:0', '0:74:0', '1:70:0', '0:69:80', '1:66:80', '2:50:80', '0:69:0', '1:66:0', '2:50:0', '0:70:80', '1:67:80', '2:55:80', '2:55:0', '2:50:80', '2:50:0', '2:46:80', '0:70:0', '1:67:0', '2:46:0', '0:72:80', '1:69:80', '2:43:80', '0:72:0', '1:69:0', '0:74:80', '1:70:80', '2:43:0', '0:74:0', '1:70:0', '0:69:80', '1:66:80', '2:50:80', '1:66:0', '1:74:80', '0:69:0', '2:50:0', '1:74:0', '1:74:80', '2:54:80', '1:74:0', '1:72:80', '2:54:0', '1:72:0', '1:72:80', '2:50:80', '1:72:0', '1:70:80', '2:50:0', '1:70:0', '1:70:80', '2:55:80', '0:79:80', '1:70:0', '2:55:0', '0:79:0', '0:79:80', '1:71:80', '2:43:80', '0:79:0', '0:77:80', '2:43:0', '0:77:0', '0:77:80', '2:55:80', '0:77:0', '0:75:80', '1:71:0', '2:55:0', '0:75:0', '0:75:80', '2:48:80', '1:72:80', '0:75:0', '2:48:0', '1:72:0', '0:76:80', '1:72:80', '2:60:80', '1:72:0', '1:70:80', '2:60:0', '1:70:0', '1:70:80', '2:48:80', '1:70:0', '1:69:80', '0:76:0', '2:48:0', '1:69:0', '1:69:80', '2:53:80', '0:77:80', '2:53:0', '0:77:0', '0:77:80', '2:41:80', '0:77:0', '0:75:80', '2:41:0', '0:75:0', '0:75:80', '2:53:80', '0:75:0', '1:69:0', '0:74:80', '2:53:0', '0:74:0', '0:74:80', '1:70:80', '2:46:80', '0:74:0', '0:74:80', '2:46:0', '0:74:0', '0:75:80', '2:43:80', '0:75:0', '0:74:80', '2:43:0', '0:74:0', '0:72:80', '2:55:80', '0:72:0', '1:70:0', '0:70:80', '2:55:0', '0:70:0', '0:72:80', '1:69:80', '2:55:80', '0:72:0', '0:72:80', '1:69:0', '2:55:0', '0:72:0', '0:74:80', '1:69:80', '2:54:80', '0:74:0', '0:72:80', '0:72:0', '0:70:80', '0:70:0', '0:69:80', '1:69:0', '2:54:0', '0:69:0', '0:70:80', '2:55:80', '1:74:80', '0:70:0', '2:55:0', '1:74:0', '0:70:80', '1:75:80', '2:43:80', '1:75:0', '1:74:80', '2:43:0', '1:74:0', '1:72:80', '2:55:80', '1:72:0', '1:70:80', '0:70:0', '2:55:0', '1:70:0', '0:69:80', '1:72:80', '2:55:80', '1:72:0', '1:72:80', '0:69:0', '2:55:0', '1:72:0', '0:69:80', '1:74:80', '2:54:80', '1:74:0', '1:72:80', '1:72:0', '1:70:80', '1:70:0', '1:69:80', '0:69:0', '2:54:0', '1:69:0', '0:67:80', '1:70:80', '2:55:80', '0:67:0', '1:70:0', '0:82:80', '1:74:80', '2:55:0', '0:82:0', '1:74:0', '0:82:80', '1:74:80', '2:43:80', '0:82:0', '1:74:0', '0:81:80', '1:72:80', '2:43:0', '0:81:0', '1:72:0', '0:81:80', '1:72:80', '2:48:80', '0:81:0', '1:72:0', '0:79:80', '1:70:80', '2:48:0', '0:79:0', '1:70:0', '0:79:80', '1:70:80', '2:50:80', '0:79:0', '1:70:0', '0:78:80', '1:69:80', '0:78:0', '1:69:0', '0:79:80', '1:70:80', '0:79:0', '1:70:0', '0:78:80', '1:69:80', '2:50:0', '0:78:0', '1:69:0', '0:79:80', '1:70:80', '0:79:0', '1:70:0', '0:78:80', '1:69:80', '2:48:80', '0:78:0', '1:69:0', '2:48:0', '0:79:80', '1:67:80', '2:46:80', '0:79:0', '0:74:80', '1:67:0', '2:46:0', '0:74:0', '0:75:80', '1:67:80', '2:48:80', '0:75:0', '1:67:0', '0:72:80', '1:69:80', '2:48:0', '0:72:0', '1:69:0', '0:70:80', '1:67:80', '2:50:80', '0:70:0', '1:67:0', '0:69:80', '1:66:80', '0:69:0', '1:66:0', '0:70:80', '1:67:80', '0:70:0', '1:67:0', '0:69:80', '1:66:80', '2:50:0', '0:69:0', '1:66:0', '0:70:80', '1:67:80', '0:70:0', '1:67:0', '0:69:80', '1:66:80', '2:38:80', '0:69:0', '1:66:0', '0:67:80', '1:67:80', '2:38:0', '0:67:0', '1:67:0', '0:67:80', '1:67:80', '2:43:80', '0:67:0', '0:70:80', '2:43:0', '0:70:0', '0:72:80', '2:44:80', '0:72:0', '2:44:0', '0:74:80', '2:46:80', '1:67:0', '0:74:0', '2:46:0', '0:75:80', '2:48:80', '0:75:0', '2:48:0', '0:77:80', '2:50:80', '0:77:0', '2:50:0', '0:79:80', '2:51:80', '1:63:80', '1:63:0', '1:67:80', '1:67:0', '1:70:80', '0:79:0', '2:51:0', '1:70:0', '1:75:80', '2:55:80', '1:75:0', '1:70:80', '2:55:0', '1:70:0', '1:72:80', '2:56:80', '1:72:0', '0:80:80', '1:84:80', '2:56:0', '0:80:0', '1:84:0', '0:79:80', '1:82:80', '2:58:80', '0:79:0', '1:82:0', '0:77:80', '1:80:80', '2:58:0', '0:77:0', '1:80:0', '0:75:80', '1:79:80', '2:46:80', '0:75:0', '1:79:0', '0:74:80', '1:77:80', '2:46:0', '0:74:0', '1:77:0', '0:75:80', '1:79:80', '2:51:80', '1:79:0', '1:63:80', '0:75:0', '1:63:0', '0:63:80', '1:67:80', '1:67:0', '1:70:80', '2:51:0', '0:63:0', '1:70:0', '0:63:80', '1:75:80', '2:55:80', '1:75:0', '1:70:80', '0:63:0', '2:55:0', '1:70:0', '0:63:80', '1:72:80', '2:56:80', '0:63:0', '1:72:0', '0:80:80', '1:84:80', '2:56:0', '0:80:0', '1:84:0', '0:79:80', '1:82:80', '2:58:80', '0:79:0', '1:82:0', '0:77:80', '1:80:80', '2:58:0', '0:77:0', '1:80:0', '0:75:80', '1:79:80', '2:46:80', '0:75:0', '1:79:0', '0:74:80', '1:77:80', '2:46:0', '0:74:0', '1:77:0', '0:75:80', '1:79:80', '2:51:80', '0:75:0', '0:70:80', '0:70:0', '0:75:80', '0:75:0', '0:79:80', '1:79:0', '2:51:0', '0:79:0', '0:82:80', '2:55:80', '0:82:0', '0:70:80', '2:55:0', '0:70:0', '0:72:80', '2:56:80', '0:72:0', '0:82:80', '2:56:0', '0:82:0', '0:80:80', '2:53:80', '0:80:0', '0:79:80', '2:53:0', '0:79:0', '0:80:80', '0:80:0', '0:72:80', '0:72:0', '0:70:80', '2:50:80', '0:70:0', '0:80:80', '2:50:0', '0:80:0', '0:79:80', '2:51:80', '0:79:0', '0:77:80', '2:51:0', '0:77:0', '0:79:80', '0:79:0', '0:70:80', '0:70:0', '0:68:80', '2:48:80', '0:68:0', '0:79:80', '2:48:0', '0:79:0', '0:77:80', '2:50:80', '0:77:0', '0:75:80', '2:50:0', '0:75:0', '0:77:80', '2:46:80', '0:77:0', '0:68:80', '2:46:0', '0:68:0', '0:67:80', '2:51:80', '1:70:80', '0:67:0', '1:70:0', '1:75:80', '1:75:0', '1:79:80', '2:51:0', '1:79:0', '1:82:80', '2:55:80', '1:82:0', '1:70:80', '2:55:0', '1:70:0', '1:72:80', '2:56:80', '1:72:0', '1:82:80', '2:56:0', '1:82:0', '1:80:80', '2:53:80', '1:80:0', '1:79:80', '2:53:0', '1:79:0', '1:80:80', '1:80:0', '1:72:80', '1:72:0', '1:70:80', '2:50:80', '1:70:0', '1:80:80', '2:50:0', '1:80:0', '1:79:80', '2:51:80', '1:79:0', '1:77:80', '2:51:0', '1:77:0', '1:79:80', '1:79:0', '1:70:80', '1:70:0', '1:68:80', '2:48:80', '1:68:0', '1:79:80', '2:48:0', '1:79:0', '1:77:80', '2:50:80', '1:77:0', '1:75:80', '2:50:0', '1:75:0', '1:77:80', '2:46:80', '1:77:0', '1:68:80', '2:46:0', '1:68:0', '1:67:80', '2:51:80', '0:70:80', '0:70:0', '0:75:80', '1:67:0', '0:75:0', '0:73:80', '1:70:80', '2:51:0', '0:73:0', '1:70:0', '0:72:80', '1:68:80', '2:55:80', '0:72:0', '1:68:0', '0:70:80', '1:67:80', '2:55:0', '0:70:0', '1:67:0', '0:72:80', '1:68:80', '2:56:80', '0:72:0', '1:68:0', '0:73:80', '1:75:80', '2:56:0', '0:73:0', '1:75:0', '0:72:80', '1:80:80', '2:53:80', '0:72:0', '1:80:0', '0:70:80', '1:79:80', '2:53:0', '0:70:0', '1:79:0', '0:68:80', '1:77:80', '0:68:0', '1:77:0', '0:67:80', '1:75:80', '0:67:0', '1:75:0', '0:65:80', '1:74:80', '2:46:80', '0:65:0', '0:70:80', '1:74:0', '0:70:0', '0:74:80', '1:70:80', '0:74:0', '0:65:80', '2:46:0', '1:70:0', '0:65:0', '0:67:80', '1:70:80', '2:51:80', '0:67:0', '0:75:80', '1:70:0', '2:51:0', '0:75:0', '0:68:80', '1:70:80', '2:50:80', '1:70:0', '1:72:80', '0:68:0', '0:67:80', '1:72:0', '1:70:80', '1:70:0', '1:72:80', '2:50:0', '0:67:0', '0:68:80', '1:72:0', '1:70:80', '2:46:80', '1:70:0', '1:72:80', '0:68:0', '0:77:80', '1:72:0', '1:70:80', '1:70:0', '1:72:80', '2:46:0', '0:77:0', '0:67:80', '1:72:0', '1:70:80', '2:51:80', '1:70:0', '1:72:80', '0:67:0', '0:75:80', '1:72:0', '1:70:80', '1:70:0', '1:72:80', '2:51:0', '0:75:0', '0:65:80', '1:72:0', '1:70:80', '2:46:80', '1:70:0', '1:72:80', '0:65:0', '0:70:80', '1:72:0', '1:70:80', '1:70:0', '1:72:80', '0:70:0', '0:74:80', '1:72:0', '1:70:80', '1:70:0', '1:72:80', '0:74:0', '0:65:80', '1:72:0', '1:70:80', '2:46:0', '1:70:0', '1:72:80', '0:65:0', '0:67:80', '1:72:0', '1:70:80', '2:51:80', '1:70:0', '1:72:80', '0:67:0', '0:75:80', '1:72:0', '1:70:80', '1:70:0', '1:72:80', '2:51:0', '0:75:0', '0:68:80', '1:72:0', '1:70:80', '2:50:80', '1:70:0', '1:72:80', '0:68:0', '0:67:80', '1:72:0', '1:70:80', '1:70:0', '1:72:80', '2:50:0', '0:67:0', '0:68:80', '1:72:0', '1:70:80', '2:46:80', '1:70:0', '1:72:80', '0:68:0', '0:77:80', '1:72:0', '1:70:80', '1:70:0', '1:72:80', '2:46:0', '0:77:0', '0:67:80', '1:72:0', '1:70:80', '2:51:80', '1:70:0', '1:72:80', '0:67:0', '0:75:80', '1:72:0', '1:70:80', '1:70:0', '1:72:80', '2:51:0', '0:75:0', '0:65:80', '1:72:0', '1:70:80', '2:46:80', '0:65:0', '0:70:80', '1:70:0', '2:46:0', '0:70:0', '0:74:80', '1:70:80', '2:58:80', '0:74:0', '1:70:0', '0:75:80', '1:72:80', '2:58:0', '0:75:0', '1:72:0', '0:77:80', '1:74:80', '2:46:80', '0:77:0', '1:74:0', '0:79:80', '1:75:80', '2:46:0', '0:79:0', '1:75:0', '0:80:80', '1:77:80', '2:46:80', '0:80:0', '1:77:0', '0:79:80', '1:75:80', '2:46:0', '0:79:0', '1:75:0', '0:77:80', '1:74:80', '2:46:80', '0:77:0', '1:74:0', '0:79:80', '1:75:80', '2:46:0', '0:79:0', '1:75:0', '0:80:80', '1:77:80', '2:46:80', '0:80:0', '1:77:0', '0:77:80', '1:74:80', '2:46:0', '0:77:0', '1:74:0', '0:79:80', '1:73:80', '2:51:80', '1:73:0', '1:72:80', '0:79:0', '1:72:0', '0:75:80', '1:73:80', '1:73:0', '1:82:80', '2:51:0', '0:75:0', '1:82:0', '0:75:80', '1:72:80', '2:56:80', '1:72:0', '1:80:80', '0:75:0', '2:56:0', '1:80:0', '0:75:80', '1:73:80', '2:55:80', '0:75:0', '0:77:80', '1:73:0', '0:77:0', '0:75:80', '1:72:80', '0:75:0', '0:77:80', '2:55:0', '1:72:0', '0:77:0', '0:75:80', '1:73:80', '2:51:80', '0:75:0', '0:77:80', '1:73:0', '0:77:0', '0:75:80', '1:82:80', '0:75:0', '0:77:80', '2:51:0', '1:82:0', '0:77:0', '0:75:80', '1:72:80', '2:56:80', '0:75:0', '0:77:80', '1:72:0', '0:77:0', '0:75:80', '1:80:80', '0:75:0', '0:77:80', '2:56:0', '1:80:0', '0:77:0', '0:75:80', '1:70:80', '2:51:80', '0:75:0', '0:77:80', '1:70:0', '0:77:0', '0:75:80', '1:75:80', '0:75:0', '0:77:80', '1:75:0', '0:77:0', '0:75:80', '1:79:80', '0:75:0', '0:77:80', '1:79:0', '0:77:0', '0:75:80', '1:70:80', '2:51:0', '0:75:0', '0:77:80', '1:70:0', '0:77:0', '0:75:80', '1:72:80', '2:56:80', '0:75:0', '0:77:80', '1:72:0', '0:77:0', '0:75:80', '1:80:80', '0:75:0', '0:77:80', '2:56:0', '1:80:0', '0:77:0', '0:75:80', '1:73:80', '2:55:80', '0:75:0', '0:77:80', '1:73:0', '0:77:0', '0:75:80', '1:72:80', '0:75:0', '0:77:80', '2:55:0', '1:72:0', '0:77:0', '0:75:80', '1:73:80', '2:51:80', '0:75:0', '0:77:80', '1:73:0', '0:77:0', '0:75:80', '1:82:80', '0:75:0', '0:77:80', '2:51:0', '1:82:0', '0:77:0', '0:75:80', '1:72:80', '2:56:80', '0:75:0', '0:77:80', '1:72:0', '0:77:0', '0:75:80', '1:80:80', '0:75:0', '0:77:80', '2:56:0', '1:80:0', '0:77:0', '0:75:80', '1:70:80', '2:51:80', '1:70:0', '1:75:80', '2:51:0', '1:75:0', '1:79:80', '2:39:80', '0:75:0', '1:79:0', '0:77:80', '1:80:80', '2:39:0', '0:77:0', '1:80:0', '0:79:80', '1:82:80', '2:51:80', '0:79:0', '1:82:0', '0:80:80', '1:84:80', '2:51:0', '0:80:0', '1:84:0', '0:82:80', '1:85:80', '2:51:80', '0:82:0', '1:85:0', '0:80:80', '1:84:80', '2:51:0', '0:80:0', '1:84:0', '0:79:80', '1:82:80', '2:51:80', '0:79:0', '1:82:0', '0:82:80', '1:85:80', '2:51:0', '0:82:0', '1:85:0', '0:80:80', '1:84:80', '2:51:80', '0:80:0', '1:84:0', '0:79:80', '1:82:80', '2:51:0', '0:79:0', '1:82:0', '0:80:80', '1:84:80', '2:56:80', '1:84:0', '1:82:80', '0:80:0', '2:56:0', '1:82:0', '0:84:80', '1:80:80', '2:56:80', '0:84:0', '1:80:0', '0:82:80', '1:79:80', '2:56:0', '0:82:0', '1:79:0', '0:80:80', '1:77:80', '2:56:80', '0:80:0', '1:77:0', '0:79:80', '1:75:80', '2:56:0', '0:79:0', '1:75:0', '0:77:80', '1:74:80', '2:58:80', '0:77:0', '1:74:0', '0:75:80', '1:75:80', '2:58:0', '0:75:0', '1:75:0', '0:74:80', '1:77:80', '2:46:80', '0:74:0', '1:77:0', '0:77:80', '1:80:80', '2:46:0', '0:77:0', '1:80:0', '0:75:80', '1:79:80', '2:46:80', '0:75:0', '1:79:0', '0:74:80', '1:77:80', '2:46:0', '0:74:0', '1:77:0', '0:75:80', '1:79:80', '2:51:80', '0:75:0', '1:79:0', '0:77:80', '1:80:80', '2:51:0', '0:77:0', '1:80:0', '0:79:80', '1:82:80', '2:51:80', '2:51:0', '2:53:80', '2:53:0', '2:55:80', '2:55:0', '2:56:80', '2:56:0', '2:58:80', '0:79:0', '1:82:0', '0:74:80', '1:77:80', '2:58:0', '0:74:0', '1:77:0', '0:74:80', '1:77:80', '2:46:80', '0:74:0', '1:77:0', '0:77:80', '1:80:80', '2:46:0', '0:77:0', '1:80:0', '0:75:80', '1:79:80', '2:58:80', '0:75:0', '1:79:0', '0:74:80', '1:77:80', '2:58:0', '0:74:0', '1:77:0', '0:75:80', '1:79:80', '2:51:80', '0:75:0', '1:79:0', '0:77:80', '1:80:80', '2:51:0', '0:77:0', '1:80:0', '0:79:80', '1:82:80', '2:51:80', '2:51:0', '2:53:80', '2:53:0', '2:55:80', '2:55:0', '2:56:80', '2:56:0', '2:58:80', '0:79:0', '1:82:0', '0:74:80', '1:77:80', '2:58:0', '0:74:0', '1:77:0', '0:74:80', '1:77:80', '2:46:80', '0:74:0', '1:77:0', '0:77:80', '1:80:80', '2:46:0', '0:77:0', '1:80:0', '0:75:80', '1:79:80', '2:58:80', '0:75:0', '1:79:0', '0:74:80', '1:77:80', '2:58:0', '0:74:0', '1:77:0', '0:75:80', '1:79:80', '2:51:80', '1:79:0', '1:80:80', '2:51:0', '1:80:0', '1:80:80', '2:53:80', '1:80:0', '1:82:80', '2:53:0', '1:82:0', '1:82:80', '2:55:80', '0:75:0', '1:82:0', '1:84:80', '2:55:0', '1:84:0', '0:75:80', '1:84:80', '2:56:80', '1:84:0', '1:74:80', '0:75:0', '0:80:80', '2:56:0', '1:74:0', '0:80:0', '0:80:80', '1:74:80', '2:58:80', '1:74:0', '1:75:80', '0:80:0', '2:58:0', '1:75:0', '0:79:80', '1:75:80', '2:60:80', '1:75:0', '1:70:80', '0:79:0', '2:60:0', '1:70:0', '0:77:80', '1:72:80', '2:56:80', '1:72:0', '1:80:80', '0:77:0', '2:56:0', '1:80:0', '0:75:80', '1:79:80', '2:58:80', '0:75:0', '1:79:0', '2:58:0', '0:74:80', '1:77:80', '2:46:80', '0:74:0', '1:77:0', '2:46:0', '0:75:80', '1:75:80', '2:51:80', '0:75:0', '1:75:0', '2:51:0', '0:70:80', '1:67:80', '2:51:80', '2:51:0', '2:53:80', '2:53:0', '2:55:80', '2:55:0', '2:56:80', '2:56:0', '2:58:80', '0:70:0', '1:67:0', '0:65:80', '1:62:80', '2:58:0', '0:65:0', '1:62:0', '0:65:80', '1:62:80', '2:46:80', '0:65:0', '1:62:0', '0:68:80', '1:65:80', '2:46:0', '0:68:0', '1:65:0', '0:67:80', '1:63:80', '2:46:80', '0:67:0', '1:63:0', '0:65:80', '1:62:80', '2:46:0', '0:65:0', '1:62:0', '0:67:80', '1:63:80', '2:51:80', '0:67:0', '1:63:0', '0:68:80', '1:65:80', '2:51:0', '0:68:0', '1:65:0', '0:70:80', '1:67:80', '2:39:80', '2:39:0', '2:41:80', '2:41:0', '2:43:80', '2:43:0', '2:44:80', '2:44:0', '2:46:80', '0:70:0', '1:67:0', '0:65:80', '1:62:80', '2:46:0', '0:65:0', '1:62:0', '0:65:80', '1:62:80', '2:46:80', '0:65:0', '1:62:0', '0:68:80', '1:65:80', '2:46:0', '0:68:0', '1:65:0', '0:67:80', '1:63:80', '2:46:80', '0:67:0', '1:63:0', '0:65:80', '1:62:80', '2:46:0', '0:65:0', '1:62:0', '0:67:80', '1:63:80', '2:51:80', '0:67:0', '0:70:80', '1:63:0', '2:51:0', '0:70:0', '0:75:80', '1:70:80', '2:55:80', '0:75:0', '0:74:80', '2:55:0', '0:74:0', '0:72:80', '2:51:80', '0:72:0', '0:70:80', '2:51:0', '0:70:0', '0:72:80', '2:44:80', '0:72:0', '0:70:80', '1:70:0', '0:70:0', '0:72:80', '1:68:80', '2:44:0', '0:72:0', '0:74:80', '2:46:80', '1:68:0', '0:74:0', '2:46:0', '0:75:80', '1:67:80', '2:48:80', '0:75:0', '0:63:80', '1:67:0', '2:48:0', '0:63:0', '0:72:80', '1:68:80', '2:44:80', '0:72:0', '1:68:0', '0:68:80', '1:65:80', '2:44:0', '0:68:0', '1:65:0', '0:67:80', '1:63:80', '2:46:80', '0:67:0', '1:63:0', '0:65:80', '1:62:80', '0:65:0', '0:67:80', '1:62:0', '1:63:80', '0:67:0', '0:65:80', '1:63:0', '1:62:80', '2:46:0', '0:65:0', '0:67:80', '1:62:0', '1:63:80', '0:67:0', '0:65:80', '1:63:0', '1:62:80', '2:46:80', '0:65:0', '0:63:80', '1:62:0', '1:63:80', '2:46:0', '0:63:0', '1:63:0', '0:63:80', '1:63:80', '2:51:80', '0:63:0', '1:63:0', '2:51:0']\n"
     ]
    }
   ],
   "source": [
    "# read out midi file as input to RNN\n",
    "import mido\n",
    "\n",
    "#outport = mido.open_output()\n",
    "niter = 0\n",
    "noteslst = []\n",
    "for msg in mido.MidiFile('bjs1031c.mid').play():\n",
    "    #outport.send(msg)\n",
    "    #print msg.bytes()\n",
    "    #print str(msg)\n",
    "    thismsg = str(msg).split(\" \") # ['note_on', 'channel=0', 'note=60', 'velocity=61', 'time=0.00168918958333']\n",
    "    if thismsg[0] == 'note_on':\n",
    "        #print thismsg\n",
    "        #print thismsg[2][5:]\n",
    "        noteslst.append(thismsg[1][8:]+':'+thismsg[2][5:]+':'+thismsg[3][9:]) # channel, note, velocity\n",
    "    niter += 1\n",
    "    if niter >= 6000:\n",
    "        break\n",
    "print(noteslst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data has 36979 characters, 14 unique.\n",
      "--full--\n",
      " 77 :54:131'409,215608 7 76782,220262,7:618,827'29923062415008:4:520,826,29'399036363741 ,,06:::7,4,,'3'4573:3 2:3:8:,'9:351 96' 2:416798 093914 650',5,17:98878 ,:,498'81447'''044'82 32277480,06 '7'::60'69238:55282 169601511559' 010'3 6882089533049:1296 3:371689:98176 02 6':81,428:6 ,4:,' '69 ,' 409993 00608'8991185'10,036'1 1741346:5'656:4155 95 91,1',452432415 19642014 95482006 '129:'3054,898, 03 31:10253'2561:6 5,456798, :85',544,51:3832 3488,3 ,44712::9107',,03083966'6,6:9:9'139868290:51:7'983'881:5479::7:03310,,'04996575637,72 7:06'5 7645:83:' ,793716,27463:,9838 244,8083,13356,948,874 '0,70 ,8:3954 ':306,1610,8:74,8'544'88:6'0 870 03530691,318,905 3 5 57354':,42'18'':15 1 ,415440955' :1415,,555 ::'055':5210,2:3 0:2:02 :2 18:53011 '4:578:26'2: ',18 ,2593:0,,34 80,3 ,09',2,0647902'2'28' \n",
      "----\n",
      "iter 0, loss: 52.781147\n",
      "--full--\n",
      " ', '0:634:0', '0:69:75:0', '24:82:780', '0:824:80:72:42:572:5:74:0', '2:0', '1:0', '1:51:5:70', '0:74:0', '0:4:60:774:80', '0:80', '2:80', '1:2:0', '0:72:0', '1:7:80', '2:434:42:79:70', '1:74:74:69:80', '1:0', '0', '1:69:80', '2:74:80', '1:0', '0:7:0', '1:0', ', '1:0', '0:72:80:46:80', '1:0', '0:72:0', '0:74:51:7:0', '1:69:870', '2:0', '0:62:0', '2:0', '1:0', '0:79:80:80', '0:74:0', '2:58:0', '2:80:72:80', '1:69:69:70', '0:72:0', '1:0', '2:0', '0:72:80', '1:72:80', '0:5:0', '2:0', '1:0', '2:0', ', '2:460:75:0', '1:72:871:69:69:72:57:0', '0:70', '1:76:80:7:80', '1:0', '2:0', '0:0', '2:4:80', '0:0', '1:80:80:79:75:75:24:2:0', '2:80', '2:534:0', '0:0', '0:75:0', '2:0', '0:80', '1:80', '0:79:72:0', '0:872:4:82:80', ', '0:74:0', '0:72:80:72:80', '0', '0', '1:72:5:56:79:5357:70:80', '0:84:0', '0 \n",
      "----\n",
      "iter 10000, loss: 12.579775\n",
      "--full--\n",
      " 80', '1:80', '0:0', '0:70', '2:0', 6:0', '0:70', '1:70', '1:70:70', '1:77:70:70', '1:70', '0:70', '0:72:8:0', '1:72:80', '1:70', '1:75:8'0:72:6:0', '1:70', '2:80', '1:72:80', '1:80:72:46:80', '2:0', '1:0', '1:0', ', '1:70', '2:80', '0:7:0', '2:89:74:0', '1:72:50', '1:72:80', '0:0', '0', '0', '1:680:77:0', '1:77:80', '0:80', '2:80', '1:76:0', '1:70', '2:0', '1:72:57:80:80', '2:80', '0:70:0', '0', '1:0', '21:80', '1:48:0', '1:70', '1:87:0', '0:75:80', '1:0', '0', '0:79:0', '0:80', '1:0', '0:70', '0:70', '0:80:70', '0:70', '1:72:51:70', '1:80:79:1', '1:72:80', '1:77:0', '2:80:80', '2:1:75:80:72:0', '1:70:80', ', '1:80', '1:62:46:0', '1:72:46:80', '1:80:70:72:80', '0', '1:80', '1:72:0', '1:74:0', '1:79:0', '1:0', :65:80', '0:7:0', '2:0', '0:72:4:80', '0:77:80', '2:0', '1:77:0', '0:77:0', '1:72 \n",
      "----\n",
      "iter 20000, loss: 12.154788\n",
      "--full--\n",
      " , '1:77:80', '0:7:0', '1:62:80', '0:77:87:80', '2:55:0', '0:70', '2:80', '1:0', '0:7:80', '0:742:461:70', '0:792:80', '1:72:1:7:80', '0:80', '0:753:80', '1:7482:8:80', 61:7:77:77:80', '0:77:80', '1:7:0', '0:72:0', '0:792:5:70', '1:80', '0:792:80', '0:80', '1:78:77:7:753480', '0:75:70', '0:80', '0:792:1:77:53:751:77:0', '0', '0:70', '0:87:87:80', '1:77:0', '0:742:52:1:61:77:80', '1:70', '0:792:8:0', '1:77:7:792:80', '1:792:80', '0:80', '1:7:0', '0:792:55:80', '1:7:0', '0:77:7:57:80', '1:792:892:80', '1:792:7:0', '1:70', '1:792:0:7:792:55:8:80', '0:7:0', '2:4461:7792:50', '2:80', '0:792:80', '0:8:0', '0:792:0', '0:792:50', '1:77:80', '0:792:51:77:87:0', '0:7:70', '2:80', '0:7:77:57:744:80', '0', '1:792:80', '0:51:77:552:80', '0:80', '0:792:50:742:80', '1:80', '1:792:80', '0:80', '1:0', '0:79 \n",
      "----\n",
      "iter 30000, loss: 14.888020\n",
      "--full--\n",
      " :0', '0:0', '2:80', '0:75:0', '0:80', '12:80', '1:80', '0:67:80', '0:75:0', '2:57:80', '2:50', '1:7:80', '0:7:80', '2:1:71:67:0', '0:75:0', '2:80', '0:80', '2:68:0', '2:53:75:80', '0:80', '1:77:80', '0:75:0', '2:50', '1:80', '1:74:0', '0:792:46:0:80', '2:80', '0:7:0', '2:50', '2:692:87:0', '2:792:5:0', '2:80', '0:77:0', '1:80', '2:80', '2:55:80', '0', '2:57:0:72:5:0:7:0', '0:70', '2:45:0', '0:7:80', '2:5:0', 6:80', '2:5:0', '2:50', '1:74:80', '2:53:70:74:85:80', '2:50', '2:80', '0:70:67:0', '2:42:12:63:0', '0:70', '2:692:74:80', '2:692:465:0', '2:0:70:70:7:80', '2:0', '2:5:7:0:74:80', '0:70:72:5:0', '1:8424:0', '2:45:0', '0:74:80', '2:5:0:74:80', '1:71:77:0', '2:57:0', '2:80', '0:80', '1:74:80', '0:42:80', '2:50', :80', '1:70', '2:80', '1:70', '0:77:0', '0:7:0', '0:0', '1:42:8:0', '0:74:0' \n",
      "----\n",
      "iter 40000, loss: 16.058110\n",
      "--full--\n",
      " :86 :80', '0:80', '1:80', '2:80', '0:7:0', '0:521:67:80', '1:7:80', '2:80', '0:80', '1:80', '0', '1:68:80', :80', '2:5:0', :80', '2:52492:524:0:7:0'0:80', '1:75:0', '0:7:0', ', '0:75:0', '1:72:5:0', '0:80', '1:80', '0', '0:7:792:80', '1:67:0', '1:792:0:80', '0', '0:7:0', '0:80', '0:75:0:70:7:80', '0:7:80', '1:76 :80', '0:45:0', '1:84:80', '0:80', '0:7:80', '1:74:0', ', '0:7265:80', '0:7:80', '0:80', '1:72:50', '0:0', '1:80', '2:51:726:80', '1:65:0', '0:75:0', '1:74:0'0:74:0', '0:70:70:7921:80', '0:80', '2:526:0', '1:67:0', '1:80', '0:7:526:80', '0:70', '1:80', '1:67:80', '1:80', '0:52:50', '1:7:80', '14:80', '2:0', '0:70:79216:80', '2:80'0:7:0', '0:74:80', '0:70:792:5:0', '1:76 :8, '0:76 :80', '2:52, '0:70:70:7:80', '1:77:80', '2:80', '2:67:0', '1:80', '1:724:0', '2:0', :80', '1:792:524:80 \n",
      "----\n",
      "iter 50000, loss: 18.385605\n",
      "--full--\n",
      " , '1:72:80', '0:80', '0:80', '0:80', '1:75:0', '1:67:7:80', '1:72:51:74:7, '2:0', '0:74:0', '0:80', '0:792:16:80', '1:80', '0:73:0', '1:72:50:792:80', '1:74:0', '1:792:0', '1:72:80', '2:53:0', '1:72:0', '2:0', '16:80', '0:53:0', '1:74:80', '2:52:0', '2:892:0', '0:70:7:80', '1:80', '2:80', '0:74:80', '1:70:70:70:80', '1:76:0', '1:80', '1:72:80', '0:74:80', '0:792:50', '2:53:0', '1:72:80', '16:80', '1:74:74:72:80', '0:7:70:80', '1:72:80', :86:0', '0:80', '1:74:0', '1:72:0', '1:74:0', '1:73:0', '1:7:0', '1:72:5:0', '2:0', :80', '0:75:0', '2:0', '1:80', '2:0', '2:0', '2:0', '1:80', '0:75:0', '1:72:51:70:70:80', '1:72:0', '0:74:0', '0:70:80', '1:80', '1:7:0:70:70:80', '1:72:892:0', '1:7:80', '0:80', '1:70:70:74:0', '1:67:80', '0:42:0', '1:68:0', '1:74:7:0', '1:42:0', '2:52:80', '1:42:0', '0:74: \n",
      "----\n",
      "iter 60000, loss: 18.401574\n"
     ]
    }
   ],
   "source": [
    "# RNN with context features (Mikolov 2015), diagonal constraints on weight matrices\n",
    "# using softmax (as in paper) as activation\n",
    "# based on Karpathys RNN\n",
    "\n",
    "import numpy as np\n",
    "import math\n",
    "\n",
    "def softmax(x):\n",
    "    ex = np.exp(x - np.max(x))\n",
    "    return ex / ex.sum()\n",
    "\n",
    "nruns = 100000\n",
    "noutput = 10000\n",
    "\n",
    "# data I/O\n",
    "data = open('notes0.txt', 'r').read() # should be simple plain text file\n",
    "chars = list(set(data))\n",
    "data_size, vocab_size = len(data), len(chars)\n",
    "print('data has %d characters, %d unique.' % (data_size, vocab_size))\n",
    "char_to_ix = { ch:i for i,ch in enumerate(chars) }\n",
    "ix_to_char = { i:ch for i,ch in enumerate(chars) }\n",
    "\n",
    "# hyperparameters\n",
    "hidden_size = 100 # size of hidden layer of neurons\n",
    "context_size = min(30, vocab_size-1) # size of hidden layer of neurons\n",
    "alpha = 0.2 # strictly between 0 and 1\n",
    "beta = np.zeros((context_size, 1))\n",
    "Q = np.zeros((context_size, context_size))\n",
    "np.fill_diagonal(Q, softmax(beta))\n",
    "seq_length = 20 # number of steps to unroll the RNN for\n",
    "learning_rate = 1e-1\n",
    "\n",
    "# model parameters\n",
    "Wxh = np.random.randn(hidden_size, vocab_size)*0.01 # input to hidden\n",
    "Whh = np.random.randn(hidden_size, hidden_size)*0.01 # hidden to hidden\n",
    "Why = np.random.randn(vocab_size, hidden_size)*0.01 # hidden to output\n",
    "bh = np.zeros((hidden_size, 1)) # hidden bias\n",
    "by = np.zeros((vocab_size, 1)) # output bias\n",
    "\n",
    "Wxs = np.random.randn(context_size, vocab_size)*0.01 # input to context: B\n",
    "#Wss = np.random.randn(context_size, context_size)*0.01 # context to context: alpha\n",
    "Wsh = np.random.randn(hidden_size, context_size)*0.01 # context to hidden: P\n",
    "Wsy = np.random.randn(vocab_size, context_size)*0.01 # context to output: V\n",
    "bys = np.zeros((vocab_size, 1)) # output context bias\n",
    "\n",
    "def lossFun(inputs, targets, hprev, sprev):\n",
    "    xs, hs, ys, ps = {}, {}, {}, {}\n",
    "    ss = {} # context\n",
    "    hs[-1] = np.copy(hprev)\n",
    "    ss[-1] = np.copy(sprev)\n",
    "    loss = 0\n",
    "    # forward pass\n",
    "    for t in range(len(inputs)):\n",
    "        xs[t] = np.zeros((vocab_size,1)) # encode in 1-of-k representation\n",
    "        xs[t][inputs[t]] = 1\n",
    "        #hs[t] = np.tanh(np.dot(Wxh, xs[t]) + np.dot(Whh, hs[t-1]) + bh) # hidden state\n",
    "        #ys[t] = np.dot(Why, hs[t]) + by # unnormalized log probabilities for next chars\n",
    "        \n",
    "        #ss[t] = 0\n",
    "        #ss[t] = np.tanh(np.dot(Wxs, xs[t]) + np.dot(Wss, ss[t-1]) + bs) # context state\n",
    "        #ss[t] = (1-alpha)*np.dot(Wxs, xs[t]) + alpha*ss[t-1] + bs # context state\n",
    "        ss[t] = np.dot((np.identity(context_size)-Q), np.dot(Wxs, xs[t])) + np.dot(Q, ss[t-1]) # context state\n",
    "        \n",
    "        hs[t] = softmax(np.dot(Wxh, xs[t]) + np.dot(Whh, hs[t-1]) + np.dot(Wsh, ss[t]) + bh) # hidden state\n",
    "        ys[t] = np.dot(Why, hs[t]) + np.dot(Wsy, ss[t]) + by # unnormalized log probabilities for next chars\n",
    "        \n",
    "        #ps[t] = np.exp(ys[t]) / (np.sum(np.exp(ys[t])) + 0.01) # probabilities for next chars\n",
    "        ps[t] = softmax(ys[t])\n",
    "        loss += -np.log(ps[t][targets[t],0]+0.00001) # softmax (cross-entropy loss)        \n",
    "        \n",
    "    # backward pass: compute gradients going backwards\n",
    "    dWxh, dWhh, dWhy = np.zeros_like(Wxh), np.zeros_like(Whh), np.zeros_like(Why)\n",
    "    dWxs, dWsh, dWsy = np.zeros_like(Wxs), np.zeros_like(Wsh), np.zeros_like(Wsy)\n",
    "    dbh, dby = np.zeros_like(bh), np.zeros_like(by)\n",
    "    dbys = np.zeros_like(bys)\n",
    "    dhnext = np.zeros_like(hs[0])\n",
    "    dvunit = np.pad(np.identity(vocab_size), ((0,hidden_size-vocab_size),(0,0)), 'constant', constant_values=(0))\n",
    "    dxhnext = dvunit * xs[0].T\n",
    "    dhunit = np.identity(hidden_size)\n",
    "    dhhnext = dhunit * hs[0].T\n",
    "    dsunit = np.pad(np.identity(context_size), ((0,hidden_size-context_size),(0,0)), 'constant', constant_values=(0))\n",
    "    dshnext = dsunit * ss[0].T\n",
    "    cvunit = np.pad(np.identity(context_size), ((0,0),(0,vocab_size-context_size)), 'constant', constant_values=(0))\n",
    "    sumt = np.zeros_like(ss[0])\n",
    "    dhtwxs = np.zeros_like(Wsh)\n",
    "    for t in reversed(range(len(inputs))):\n",
    "        dy = np.copy(ps[t])\n",
    "        dy[targets[t]] -= 1 # backprop into y. see http://cs231n.github.io/neural-networks-case-study/#grad if confused here\n",
    "        dWhy += np.dot(dy, hs[t].T)\n",
    "        dWsy += np.dot(dy, ss[t].T)\n",
    "        dby += dy\n",
    "        dh = np.dot(Why.T, dy) + dhnext # backprop into h\n",
    "\n",
    "        dhraw = hs[t] * (1 - hs[t]) # backprop through tanh nonlinearity\n",
    "        dsraw = ss[t] * (1 - ss[t])\n",
    "        whhdrwa = np.dot(Whh, dhraw)\n",
    "        \n",
    "        dxhnext = dvunit * xs[t].T + whhdrwa * dxhnext\n",
    "        dWxh += np.dot(dxhnext.T, dh).T\n",
    "        \n",
    "        dbh += dhraw * dh\n",
    "        \n",
    "        dhhnext = dhunit * hs[t-1].T + whhdrwa * dhhnext\n",
    "        dWhh += np.dot(dhhnext.T, dh).T\n",
    "        \n",
    "        sumt = np.dot((np.identity(context_size)-Q), np.dot(cvunit, xs[t])) + np.dot(Q, sumt)\n",
    "        #dhtwxs = np.dot(Whh, dhraw) * np.dot(Wsh, (dhtwxs + sumt))\n",
    "        #print(np.shape(Wsh * sumt.T))\n",
    "        dhtwxs = whhdrwa * (dhtwxs + Wsh * sumt.T)\n",
    "        \n",
    "        #print(np.shape(dhtwxs))\n",
    "        #print(np.shape(np.dot(Why, dhtwxs)))\n",
    "        #print(np.shape(Wsy * sumt.T))\n",
    "        dWxs += (dy * (np.dot(Why, dhtwxs) + (Wsy * sumt.T))).T\n",
    "\n",
    "        dshnext = whhdrwa * dshnext + dsunit * ss[t].T\n",
    "        dWsh += dh * dshnext\n",
    "        \n",
    "        dhnext = np.dot(Whh.T, dhraw)\n",
    "        dsnext = np.dot(Q.T, dsraw)\n",
    "        \n",
    "    for dparam in [dWxh, dWhh, dWhy, dbh, dby, dWxs, dWsh, dWsy, dbys]:\n",
    "        np.clip(dparam, -5, 5, out=dparam) # clip to mitigate exploding gradients\n",
    "        clip_diag(dparam, 1)\n",
    "    return loss, dWxh, dWhh, dWhy, dbh, dby, hs[len(inputs)-1], dWxs, dWsh, dWsy, dbys, ss[len(inputs)-1]\n",
    "\n",
    "def sample(h, s, seed_ix, n, context):\n",
    "    x = np.zeros((vocab_size, 1))\n",
    "    x[seed_ix] = 1\n",
    "    ixes = []\n",
    "    for t in range(n):\n",
    "        #s = np.tanh(np.dot(Wxs, x) + np.dot(Wss, s) + bs) # context state\n",
    "        #s = (1-alpha)*np.dot(Wxs, x) + alpha*s + bs # context state\n",
    "        s = np.dot((np.identity(context_size)-Q), np.dot(Wxs, x)) + np.dot(Q, s) # context state\n",
    "        h = softmax(np.dot(Wxh, x) + np.dot(Whh, h) + np.dot(Wsh, s) + bh)\n",
    "        if context:\n",
    "            y = np.dot(Wsy, s) + by # only context\n",
    "        else:\n",
    "            y = np.dot(Why, h) + np.dot(Wsy, s) + by\n",
    "        p = softmax(y)\n",
    "        pl = p.ravel()\n",
    "        ix = np.random.choice(range(vocab_size), p=pl)\n",
    "        x = np.zeros((vocab_size, 1))\n",
    "        x[ix] = 1\n",
    "        ixes.append(ix)\n",
    "    return ixes\n",
    "\n",
    "def clip_diag(w, cval):\n",
    "    n = min(np.shape(w))\n",
    "    w[range(n), range(n)] = np.clip(np.diagonal(w), -cval, cval)\n",
    "\n",
    "n, p = 0, 0\n",
    "mWxh, mWhh, mWhy = np.zeros_like(Wxh), np.zeros_like(Whh), np.zeros_like(Why)\n",
    "mWxs, mWsh, mWsy = np.zeros_like(Wxs), np.zeros_like(Wsh), np.zeros_like(Wsy)\n",
    "mbh, mby = np.zeros_like(bh), np.zeros_like(by) # memory variables for Adagrad\n",
    "mbys = np.zeros_like(bys) # memory context bias\n",
    "smooth_loss = -np.log(1.0/vocab_size)*seq_length # loss at iteration 0\n",
    "#while True:\n",
    "while n<nruns:\n",
    "    # prepare inputs (we're sweeping from left to right in steps seq_length long)\n",
    "    if p+seq_length+1 >= len(data) or n == 0: \n",
    "        hprev = np.zeros((hidden_size, 1)) # reset RNN memory\n",
    "        sprev = np.zeros((context_size, 1)) # reset context\n",
    "        p = 0 # go from start of data\n",
    "    inputs = [char_to_ix[ch] for ch in data[p:p+seq_length]]\n",
    "    targets = [char_to_ix[ch] for ch in data[p+1:p+seq_length+1]]\n",
    "\n",
    "    # sample from the model now and then\n",
    "    if n % noutput == 0:\n",
    "        sample_ix = sample(hprev, sprev, inputs[0], 800, False)\n",
    "        txt = ''.join(ix_to_char[ix] for ix in sample_ix)\n",
    "        print('--full--\\n %s \\n----' % (txt, ))\n",
    "        #sample_ix = sample(hprev, sprev, inputs[0], 300, True)\n",
    "        #txt = ''.join(ix_to_char[ix] for ix in sample_ix)\n",
    "        #print('--context--\\n %s \\n----' % (txt, ))\n",
    "\n",
    "    # forward seq_length characters through the net and fetch gradient\n",
    "    loss, dWxh, dWhh, dWhy, dbh, dby, hprev, dWxs, dWsh, dWsy, dbys, sprev = lossFun(inputs, targets, hprev, sprev)\n",
    "    smooth_loss = smooth_loss * 0.999 + loss * 0.001\n",
    "    if n % noutput == 0:\n",
    "        print('iter %d, loss: %f' % (n, smooth_loss)) # print progress\n",
    "  \n",
    "    # perform parameter update with Adagrad\n",
    "    for param, dparam, mem in zip([Wxh, Whh, Why, bh, by, Wxs, Wsh, Wsy, bys], \n",
    "                                    [dWxh, dWhh, dWhy, dbh, dby, dWxs, dWsh, dWsy, dbys], \n",
    "                                    [mWxh, mWhh, mWhy, mbh, mby, mWxs, mWsh, mWsy, mbys]):\n",
    "        mem += dparam * dparam\n",
    "        param += -learning_rate * dparam / np.sqrt(mem + 1e-8) # adagrad update\n",
    "\n",
    "    p += seq_length # move data pointer\n",
    "    n += 1 # iteration counter "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}