music-transformer-train.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"train.ipynb","version":"0.3.2","provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"id":"vT_Gdqetn8XT","colab_type":"code","outputId":"1dd6111f-dc01-479e-a423-cd38f74cc147","executionInfo":{"status":"ok","timestamp":1559096165648,"user_tz":-540,"elapsed":19211,"user":{"displayName":"양기창","photoUrl":"https://lh3.googleusercontent.com/-4HfYuDxbUNQ/AAAAAAAAAAI/AAAAAAAAAD8/1C9xKnO2Ji8/s64/photo.jpg","userId":"09400683122953982681"}},"colab":{"base_uri":"https://localhost:8080/","height":122}},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n","\n","Enter your authorization code:\n","··········\n","Mounted at /content/drive\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"IiDyGS9DoUme","colab_type":"code","outputId":"64fd0568-b3a3-46f3-90da-c8ab10171411","executionInfo":{"status":"ok","timestamp":1559096165661,"user_tz":-540,"elapsed":3623,"user":{"displayName":"양기창","photoUrl":"https://lh3.googleusercontent.com/-4HfYuDxbUNQ/AAAAAAAAAAI/AAAAAAAAAD8/1C9xKnO2Ji8/s64/photo.jpg","userId":"09400683122953982681"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["cd /content/drive/'My Drive'/MusicTransformer"],"execution_count":2,"outputs":[{"output_type":"stream","text":["/content/drive/My Drive/MusicTransformer\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"ttZxh2SyKFj8","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":85},"outputId":"9447bf43-13a4-4bf0-d73e-58b19e287d11","executionInfo":{"status":"ok","timestamp":1559096167437,"user_tz":-540,"elapsed":4749,"user":{"displayName":"양기창","photoUrl":"https://lh3.googleusercontent.com/-4HfYuDxbUNQ/AAAAAAAAAAI/AAAAAAAAAD8/1C9xKnO2Ji8/s64/photo.jpg","userId":"09400683122953982681"}}},"source":["ls"],"execution_count":3,"outputs":[{"output_type":"stream","text":["\u001b[0m\u001b[01;34mbin\u001b[0m/      model.py   preprocess.py     seq_test.py  train.ipynb\n","\u001b[01;34mcustom\u001b[0m/   out.mid    \u001b[01;34m__pycache__\u001b[0m/      sequence.py  train.py\n","data.py   out.midi   README.md         \u001b[01;34mtestdata\u001b[0m/    utils.py\n","\u001b[01;34mdataset\u001b[0m/  params.py  requirements.txt  test.mid\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"DaY3Po_wLDYC","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":1211},"outputId":"fb86591f-acd3-4633-c4c1-d6f840ddf2ad","executionInfo":{"status":"ok","timestamp":1559096219508,"user_tz":-540,"elapsed":55023,"user":{"displayName":"양기창","photoUrl":"https://lh3.googleusercontent.com/-4HfYuDxbUNQ/AAAAAAAAAAI/AAAAAAAAAD8/1C9xKnO2Ji8/s64/photo.jpg","userId":"09400683122953982681"}}},"source":["!pip install -r requirements.txt"],"execution_count":4,"outputs":[{"output_type":"stream","text":["Requirement already satisfied: absl-py==0.7.1 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 1)) (0.7.1)\n","Requirement already satisfied: astor==0.8.0 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 2)) (0.8.0)\n","Collecting config==0.4.2 (from -r requirements.txt (line 3))\n","  Downloading https://files.pythonhosted.org/packages/59/6c/4ab0d80b22dca3baab49670b75ae2183b59649e9f27c11018075e509048e/config-0.4.2.tar.gz\n","Requirement already satisfied: gast==0.2.2 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 4)) (0.2.2)\n","Collecting google-pasta==0.1.6 (from -r requirements.txt (line 5))\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f9/68/a14620bfb042691f532dcde8576ff82ee82e4c003cdc0a3dbee5f289cee6/google_pasta-0.1.6-py3-none-any.whl (51kB)\n","\u001b[K     |████████████████████████████████| 61kB 7.9MB/s \n","\u001b[?25hCollecting grpcio==1.21.1 (from -r requirements.txt (line 6))\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/99/83/18f374294bf34128a448ee2fae37651f943b0b5fa473b5b3aff262c15bf8/grpcio-1.21.1-cp36-cp36m-manylinux1_x86_64.whl (2.2MB)\n","\u001b[K     |████████████████████████████████| 2.2MB 14.4MB/s \n","\u001b[?25hCollecting h5py==2.9.0 (from -r requirements.txt (line 7))\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/30/99/d7d4fbf2d02bb30fb76179911a250074b55b852d34e98dd452a9f394ac06/h5py-2.9.0-cp36-cp36m-manylinux1_x86_64.whl (2.8MB)\n","\u001b[K     |████████████████████████████████| 2.8MB 47.6MB/s \n","\u001b[?25hRequirement already satisfied: Keras-Applications==1.0.7 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 8)) (1.0.7)\n","Requirement already satisfied: Keras-Preprocessing==1.0.9 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 9)) (1.0.9)\n","Requirement already satisfied: Markdown==3.1.1 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 10)) (3.1.1)\n","Collecting mido==1.2.9 (from -r requirements.txt (line 11))\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/20/0a/81beb587b1ae832ea6a1901dc7c6faa380e8dd154e0a862f0a9f3d2afab9/mido-1.2.9-py2.py3-none-any.whl (52kB)\n","\u001b[K     |████████████████████████████████| 61kB 27.6MB/s \n","\u001b[?25hCollecting numpy==1.16.4 (from -r requirements.txt (line 12))\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/87/2d/e4656149cbadd3a8a0369fcd1a9c7d61cc7b87b3903b85389c70c989a696/numpy-1.16.4-cp36-cp36m-manylinux1_x86_64.whl (17.3MB)\n","\u001b[K     |████████████████████████████████| 17.3MB 40.2MB/s \n","\u001b[?25hRequirement already satisfied: pretty-midi==0.2.8 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 13)) (0.2.8)\n","Collecting progress==1.5 (from -r requirements.txt (line 14))\n","  Downloading https://files.pythonhosted.org/packages/38/ef/2e887b3d2b248916fc2121889ce68af8a16aaddbe82f9ae6533c24ff0d2b/progress-1.5.tar.gz\n","Requirement already satisfied: protobuf==3.7.1 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 15)) (3.7.1)\n","Requirement already satisfied: six==1.12.0 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 16)) (1.12.0)\n","Collecting tb-nightly==1.14.0a20190301 (from -r requirements.txt (line 17))\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a9/51/aa1d756644bf4624c03844115e4ac4058eff77acd786b26315f051a4b195/tb_nightly-1.14.0a20190301-py3-none-any.whl (3.0MB)\n","\u001b[K     |████████████████████████████████| 3.0MB 31.7MB/s \n","\u001b[?25hCollecting tensorflow-gpu==2.0.0a0 (from -r requirements.txt (line 18))\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1a/66/32cffad095253219d53f6b6c2a436637bbe45ac4e7be0244557210dc3918/tensorflow_gpu-2.0.0a0-cp36-cp36m-manylinux1_x86_64.whl (332.1MB)\n","\u001b[K     |████████████████████████████████| 332.1MB 53kB/s \n","\u001b[?25hRequirement already satisfied: termcolor==1.1.0 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 19)) (1.1.0)\n","Collecting tf-estimator-nightly==1.14.0.dev2019030115 (from -r requirements.txt (line 20))\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/13/82/f16063b4eed210dc2ab057930ac1da4fbe1e91b7b051a6c8370b401e6ae7/tf_estimator_nightly-1.14.0.dev2019030115-py2.py3-none-any.whl (411kB)\n","\u001b[K     |████████████████████████████████| 419kB 45.1MB/s \n","\u001b[?25hRequirement already satisfied: Werkzeug==0.15.4 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 21)) (0.15.4)\n","Requirement already satisfied: setuptools>=36 in /usr/local/lib/python3.6/dist-packages (from Markdown==3.1.1->-r requirements.txt (line 10)) (41.0.1)\n","Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tb-nightly==1.14.0a20190301->-r requirements.txt (line 17)) (0.33.4)\n","Building wheels for collected packages: config, progress\n","  Building wheel for config (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Stored in directory: /root/.cache/pip/wheels/51/7d/db/0e38d2ec57843d00cc39f8df3686984ccec689694f7bc78a38\n","  Building wheel for progress (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Stored in directory: /root/.cache/pip/wheels/6c/c8/80/32a294e3041f006c661838c05a411c7b7ffc60ff939d14e116\n","Successfully built config progress\n","\u001b[31mERROR: magenta 0.3.19 has requirement mido==1.2.6, but you'll have mido 1.2.9 which is incompatible.\u001b[0m\n","\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n","\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n","Installing collected packages: config, google-pasta, grpcio, numpy, h5py, mido, progress, tb-nightly, tf-estimator-nightly, tensorflow-gpu\n","  Found existing installation: grpcio 1.15.0\n","    Uninstalling grpcio-1.15.0:\n","      Successfully uninstalled grpcio-1.15.0\n","  Found existing installation: numpy 1.16.3\n","    Uninstalling numpy-1.16.3:\n","      Successfully uninstalled numpy-1.16.3\n","  Found existing installation: h5py 2.8.0\n","    Uninstalling h5py-2.8.0:\n","      Successfully uninstalled h5py-2.8.0\n","  Found existing installation: mido 1.2.6\n","    Uninstalling mido-1.2.6:\n","      Successfully uninstalled mido-1.2.6\n","Successfully installed config-0.4.2 google-pasta-0.1.6 grpcio-1.21.1 h5py-2.9.0 mido-1.2.9 numpy-1.16.4 progress-1.5 tb-nightly-1.14.0a20190301 tensorflow-gpu-2.0.0a0 tf-estimator-nightly-1.14.0.dev2019030115\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.colab-display-data+json":{"pip_warning":{"packages":["grpc","numpy"]}}},"metadata":{"tags":[]}}]},{"cell_type":"code","metadata":{"id":"WOW0s-pVngrO","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"outputId":"bf805741-7c30-4c0d-aea9-0010aa8d17a6","executionInfo":{"status":"ok","timestamp":1559096227680,"user_tz":-540,"elapsed":62073,"user":{"displayName":"양기창","photoUrl":"https://lh3.googleusercontent.com/-4HfYuDxbUNQ/AAAAAAAAAAI/AAAAAAAAAD8/1C9xKnO2Ji8/s64/photo.jpg","userId":"09400683122953982681"}}},"source":["from model import MusicTransformer\n","from custom.layers import *\n","from custom import callback\n","from tensorflow.python import keras\n","import params as par\n","from tensorflow.python import enable_eager_execution\n","from tensorflow.python.keras.optimizer_v2.adam import Adam\n","from data import Data\n","import utils\n","tf.executing_eagerly()"],"execution_count":5,"outputs":[{"output_type":"execute_result","data":{"text/plain":["True"]},"metadata":{"tags":[]},"execution_count":5}]},{"cell_type":"code","metadata":{"id":"qhPpF77qngrT","colab_type":"code","cellView":"form","colab":{}},"source":["l_r = 0.0024 #@param {type:\"slider\", min:0, max:0.1, step:0.0001}\n","batch_size = 3 #@param {type:\"slider\", min:1, max:100, step:1}\n","pickle_dir = \"processed/\" #@param {type:\"string\"}\n","max_seq = 2048 #@param {type:\"slider\", min:1, max:3000, step:1}\n","epochs = 1416 #@param {type:\"slider\", min:1, max:10000, step:1}\n","model_save_path = \"bin/models\" #@param {type:\"string\"}"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"S3vb_7QdngrW","colab_type":"code","colab":{}},"source":["dataset = Data('dataset/processed/')\n","opt = Adam(par.l_r)\n","mt = MusicTransformer(\n","    embedding_dim=par.embedding_dim, \n","    vocab_size=par.vocab_size, \n","    num_layer=6, \n","    max_seq=2048,\n","    debug=False)\n","mt.compile(optimizer=opt, loss=callback.TransformerLoss())"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"DV-qQqe_TM3U","colab_type":"text"},"source":["# Train Model"]},{"cell_type":"code","metadata":{"id":"ghmPk2DsTPv1","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":422},"outputId":"29cf9a1f-6eda-42aa-e935-ed8816b3c0c8","executionInfo":{"status":"error","timestamp":1559096593998,"user_tz":-540,"elapsed":2658,"user":{"displayName":"양기창","photoUrl":"https://lh3.googleusercontent.com/-4HfYuDxbUNQ/AAAAAAAAAAI/AAAAAAAAAD8/1C9xKnO2Ji8/s64/photo.jpg","userId":"09400683122953982681"}}},"source":["    for e in range(epochs):\n","        for b in range(len(dataset.files) // batch_size):\n","            batch_x, batch_y = dataset.seq2seq_batch(batch_size, par.max_seq)\n","            result_metrics = mt.train_on_batch(batch_x, batch_y)\n","            if b % 100 == 0:\n","              print('===========================================\\n')\n","              print('Loss: {:6.6}, Accuracy: {:3.2}'.format(result_metrics[0], result_metrics[1]))\n","              mt.save('mt-2048-h4-dim256.h5', )\n","              \n","            "],"execution_count":11,"outputs":[{"output_type":"stream","text":["===========================================\n","\n","Loss: 5.81973, Accuracy: 9.2\n"],"name":"stdout"},{"output_type":"error","ename":"NotImplementedError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNotImplementedError\u001b[0m                       Traceback (most recent call last)","\u001b[0;32m<ipython-input-11-de26e9408e62>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m           \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'===========================================\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m           \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Loss: {:6.6}, Accuracy: {:3.2}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult_metrics\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult_metrics\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m           \u001b[0mmt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'mt-2048-h4-dim256.h5'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, filepath, overwrite, include_optimizer)\u001b[0m\n\u001b[1;32m   1305\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_graph_network\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1306\u001b[0m       raise NotImplementedError(\n\u001b[0;32m-> 1307\u001b[0;31m           \u001b[0;34m'The `save` method requires the model to be a Functional model or a '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1308\u001b[0m           \u001b[0;34m'Sequential model. It does not work for subclassed models, '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1309\u001b[0m           \u001b[0;34m'because such models are defined via the body of a Python method, '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mNotImplementedError\u001b[0m: The `save` method requires the model to be a Functional model or a Sequential model. It does not work for subclassed models, because such models are defined via the body of a Python method, which isn't safely serializable. Consider using `save_weights`, in order to save the weights of the model."]}]},{"cell_type":"markdown","metadata":{"id":"OUye7rPRngrb","colab_type":"text"},"source":["# Generate Model"]},{"cell_type":"code","metadata":{"id":"wzvXA7iKngrb","colab_type":"code","colab":{}},"source":["def fill_with_placeholder(prev_data: list, max_len: int, max_val: int=239):\n","    placeholder = [max_val for _ in range(max_len - len(prev_data))]\n","    return placeholder+prev_data"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"m5jlJW2Vngrd","colab_type":"code","colab":{}},"source":["for i in range(max_seq):\n","    "],"execution_count":0,"outputs":[]}]}