train_cpn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "39419753",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import logging\n",
    "import os\n",
    "import sys\n",
    "\n",
    "l = logging.getLogger()\n",
    "lh = logging.StreamHandler(sys.stdout)\n",
    "l.addHandler(lh)\n",
    "l.setLevel(logging.INFO)\n",
    "\n",
    "import torch\n",
    "\n",
    "import experiment\n",
    "import cpn\n",
    "\n",
    "# Vanilla case\n",
    "#cfg = experiment.experiment.get_config(recover_after_lesion=False, coadapt=False, cuda=\"3\")\n",
    "\n",
    "# Co-adaptation case\n",
    "cfg = experiment.experiment.get_config(recover_after_lesion=True, coadapt=True, cuda=\"0\")\n",
    "\n",
    "# Co-adaptation with an AIP lesion\n",
    "#cfg = experiment.experiment.get_aip_lesion_config(cuda=\"3\", coadapt=False)\n",
    "\n",
    "# M1 lesion case\n",
    "#cfg = experiment.experiment.get_m1_lesion_config(cuda=\"3\")\n",
    "\n",
    "\n",
    "LOG_DIR = \"/home/mbryan/coproc-poc/models\"\n",
    "my_coproc = cpn.CPN_EN_CoProc(cfg, log_dir=LOG_DIR, recycle_thresh=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c3b740a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "my_experiment = experiment.experiment.stage(my_coproc, cfg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0042fe03",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Start running experiment; this and the next cell can be killed with\n",
    "#  ctrl+c, and resumed with the cell below.\n",
    "\n",
    "loss_history = my_experiment.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "208288fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Show me how I did, make pretty graphs, etc.\n",
    "# TODO: this.\n",
    "#experiment.score(my_experiment.loss_history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "52a1caef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAco0lEQVR4nO3df5Bd5X3f8fdnd7UyCFsCsbiglVkZyXEFpoC3FByPB6NCRBKQhx9jadqCY1Iorjp1adqIodbUWDMOrRNsjxkTjaEhuOWXgGRjlKiJhZOBxLJWAgMCBCsFLAkbrQSW+VEhif32j/Msunv3Lrrs3r3n3D2f18ydPee5zz33e3bP3u99fpxzFBGYmVn5tOUdgJmZ5cMJwMyspJwAzMxKygnAzKyknADMzEqqI+8A3o/jjz8+enp68g7DzKylbNq0aU9EdFWXt1QC6Onpob+/P+8wzMxaiqSXapW7C8jMrKScAMzMSsoJwMyspJwAzMxKygnAzKyknADMzErKCcDMrKRa6jwAM7OpaN//O8j3f/wSbx98Z8w6/2HRAqa1N/Y7uxOAmVnOfrR1N/9z3VYApNp1vvTZ+Uxrb+z7OgGYmeXsnaHsxlx/+1/O4+TZM5r2vh4DMDPLWV43ZnQCMDPL2fDnvxij/2eSOAGYmZWUE4CZWc4i9QGNNQA8WZwAzKxpXnjldW548EmGhnLq9LYRnADMrGmuvWsTd/9kB9v3vJl3KIWSVzp0AjCz5nm3i8MtgCJwAjCzpmlyF3frSPnQYwBmNuXlNe/dRqorAUhaLGmrpAFJK2o8P13Sven5DZJ6UvkFkjZJeir9PL/iNctS+ZOS/krS8Q3bKzMrJKWvuP78HykYngVUsPMAJLUDtwIXAQuBZZIWVlW7GngtIuYDtwA3p/I9wMUR8QngKuCutM0O4FvAZyPidOBJYPnEd8fMisxdQO+t2b+feloAZwMDEbE9Ig4A9wBLquosAe5My2uARZIUEY9HxMupfAtwlKTpZPspYIaylPch4GXMbEob/oLrLqCRinwpiDnAjor1namsZp2IOATsA2ZX1bkM2BwRb0fEQeA64CmyD/6FwO3vO3ozaynDlzoYcgaoaUoOAks6laxb6Nq0Po0sAZwJnETWBXTDGK+9RlK/pP7BwcFmhGtmk6TZH3CtosjnAewC5lasd6eymnVS//5MYG9a7wYeAq6MiG2p/hkAEbEtsnOg7wM+VevNI2J1RPRGRG9XV1c9+2RmBecGwEjDv48iXgxuI7BA0jxJncBSoK+qTh/ZIC/A5cD6iAhJs4CHgRUR8VhF/V3AQknDn+gXAM+Ocx/MrMWE5wEVwhFvCBMRhyQtB9YB7cAdEbFF0k1Af0T0kfXf3yVpAHiVLElANrNnPrBS0spUdmFEvCzpq8DfSToIvAR8oZE7ZmbF0+xpjq3i8DTQ5r5vXXcEi4i1wNqqspUVy/uBK2q8bhWwaoxt3gbc9n6CNbOpwV1AxeAzgc2sadrcAKjp8BhAczkBmFnT+DyAYnECMLOm8yDwSO/+NqbieQBmZtD8aY6tpojTQM3MGspdQFVy+oU4AZhZ07w7BpBvGIU1JS8FYWYGvhroWIp8KQgzs8YYvh+A+4BG8DRQM5vyhj/ghvz5XwhOAGbWNL4SRG3DLaLC3RHMzKzx3AQogrquBWRm1gjD32+/s36AOccelWssRfI3z+wGmj8G4ARgZk1zStcxbP7ZL/nx9lc5urM973AKYe+bB3J7bycAM2uaU044BoDNX7mAo5wAAPj+j1/iv/3Z04DPAzAzK5XKD31fCsLMpixP/x8tz+sj1ZUAJC2WtFXSgKQVNZ6fLune9PwGST2p/AJJmyQ9lX6eX/GaTkmrJT0v6TlJlzVsr8ys0Dwd9LARv4ui3RFMUjtwK9l9e3cCGyX1RcQzFdWuBl6LiPmSlgI3A58H9gAXp1tAnkZ2W8k56TU3Arsj4mOS2oDjGrZXZmYtIs9cWE8L4GxgICK2R8QB4B5gSVWdJcCdaXkNsEiSIuLxiHg5lW8BjpI0Pa1/Efg6QEQMRcSeieyImRWf7wMw2ogxgAIOAs8BdlSs7+Twt/hRdSLiELAPmF1V5zJgc0S8LWlWKvuapM2S7pf04VpvLukaSf2S+gcHB+sI18ysdRR+DGCiJJ1K1i10bSrqALqBv4+Is4B/AL5R67URsToieiOit6urqxnhmtkk8SBwDaq52BT1JIBdwNyK9e5UVrOOpA5gJrA3rXcDDwFXRsS2VH8v8BbwYFq/HzhrHPGbWQvyIPBhRR8D2AgskDRPUiewFOirqtMHXJWWLwfWR0Skrp6HgRUR8dhw5ciufPQXwHmpaBFQOahsZlYKlReAK9zF4FKf/nKyGTzPAvdFxBZJN0m6JFW7HZgtaQC4HhieKrocmA+slPREepyQnvt94L9LehL4N8B/bthemZm1iDxbAHVdCiIi1gJrq8pWVizvB66o8bpVwKoxtvkS8Jn3E6yZTQ2+OfxhKvgYgJmZTZKiTwM1M2sI3wpytCk/DdTMrJJnAR3mi8GZmVnTOQGYWdO4B2i0kdNAm/veTgBm1nTuATqs6CeCmZnZJMlzPMQJwMyaxj1Ao3kWkJmVSrMveVBkPg/AzKykPAZgZqXgWUCj+TwAMysVdwBV8jRQMysB3xJyNM8CMrNS8RjwYRpjuRmcAMzMcpTnjCgnADNrGg8CjzaiBVC0O4IBSFosaaukAUkrajw/XdK96fkNknpS+QWSNkl6Kv08v8Zr+yQ9PeE9MbOW4fMADiv0GICkduBW4CJgIbBM0sKqalcDr0XEfOAW4OZUvge4OCI+QXbP4Luqtn0p8MaE9sDMrIUV/Y5gZwMDEbE9Ig4A9wBLquosAe5My2uARZIUEY9HxMupfAtwlKTpAJKOIbt/cM1bRprZ1OMeoNGKfimIOcCOivWdqaxmnXQT+X3A7Ko6lwGbI+LttP414A+Bt97rzSVdI6lfUv/g4GAd4ZqZtZCpfikISaeSdQtdm9bPAE6JiIeO9NqIWB0RvRHR29XVNbmBmpk1WdEvBbELmFux3p3KataR1AHMBPam9W7gIeDKiNiW6p8L9Ep6EXgU+JikH41vF8ysZXga0CgjbwhTvFlAG4EFkuZJ6gSWAn1VdfrIBnkBLgfWR0RImgU8DKyIiMeGK0fEdyPipIjoAT4NPB8R501oT8ysJXgC0EiFbgGkPv3lwDrgWeC+iNgi6SZJl6RqtwOzJQ2QDewOTxVdDswHVkp6Ij1OaPhemJm1qDwTYkc9lSJiLbC2qmxlxfJ+4Ioar1vFEWb5RMSLwGn1xGFmrc0dQKMVfRaQmVnDuAdopEKfCGZm1igeAx6t0GMAZmaN5MtAVHELwMysnDwGYGal4BvCjOYxADMrDXcAjeQxADOzkvINYcysFDwLaDR3AZlZaXgS0EjuAjIzKym3AMysFNwDVIvHAMysJPKc915EbgGYmZWUxwDMrBQ8C2g0TwM1s/JwD9AIbgGYmZVUW9FbAJIWS9oqaUDSihrPT5d0b3p+g6SeVH6BpE2Snko/z0/lR0t6WNJzkrZI+oOG7pWZFZKvBTRaoQeBJbUDtwIXAQuBZZIWVlW7GngtIuYDtwA3p/I9wMUR8QmyewbfVfGab0TEx4EzgV+XdNGE9sTMWoJ7gIqjnhbA2cBARGyPiAPAPcCSqjpLgDvT8hpgkSRFxOMR8XIq3wIcJWl6RLwVEY8ApG1uBronujNmVnBuAIxS6BYAMAfYUbG+M5XVrJNuIr8PmF1V5zJgc0S8XVkoaRZwMfDDWm8u6RpJ/ZL6BwcH6wjXzIrMl4IYacrfD0DSqWTdQtdWlXcAdwPfjojttV4bEasjojcieru6uiY/WDOzJip6C2AXMLdivTuV1ayTPtRnAnvTejfwEHBlRGyret1q4IWI+Ob7jtzMWo57gEYregLYCCyQNE9SJ7AU6Kuq00c2yAtwObA+IiJ17zwMrIiIxypfIGkVWaL48vjDN7NW40tBjFToLqDUp78cWAc8C9wXEVsk3STpklTtdmC2pAHgemB4quhyYD6wUtIT6XFCahXcSDaraHMq/93G7pqZWfHl2QLoqKdSRKwF1laVraxY3g9cUeN1q4BVY2zWXwPMSiZ8LYhRfCawmZWGZwGNVPQxADMzmzQFHgMwM2sU9wCN5haAmZWGe4BG8hiAmVlJ+X4AZlYK7gEazS0AMyuNPL/xFpHHAMzMSqrQZwKbmTWKZwGN5haAmZWGO4BGcgIws1LwLSFH8ywgMysPNwFG8CwgM7OScheQmZWCB4FH8ywgMysN9wCNVPgWgKTFkrZKGpC0osbz0yXdm57fIKknlV8gaZOkp9LP8yte88lUPiDp2/LZIWZWQoUeA5DUDtwKXER2B69lkhZWVbsaeC0i5gO3kN0AHmAPcHFEfILslpF3Vbzmu8C/BRakx+IJ7IeZWWsqeAvgbGAgIrZHxAHgHmBJVZ0lwJ1peQ2wSJIi4vGIeDmVbwGOSq2FE4EPRcSPI7tF0J8Cn5vozphZ8bmxP1LRxwDmADsq1nemspp10j2E9wGzq+pcBmyOiLdT/Z1H2KaZ2ZRX+HsCT5SkU8m6hS4cx2uvAa4B+MhHPtLgyMysmXxP4NEKPQYA7ALmVqx3p7KadSR1ADOBvWm9G3gIuDIitlXU7z7CNgGIiNUR0RsRvV1dXXWEa2ZFFUCbe4BGKPqZwBuBBZLmSeoElgJ9VXX6yAZ5AS4H1kdESJoFPAysiIjHhitHxM+BX0k6J83+uRL484ntipkV3VCExwCqFLoFkPr0lwPrgGeB+yJii6SbJF2Sqt0OzJY0AFwPDE8VXQ7MB1ZKeiI9TkjPfQn4HjAAbAP+slE7ZWbFFOEWQLXCjwFExFpgbVXZyorl/cAVNV63Clg1xjb7gdPeT7Bm1tqGAnwq2EhFnwVkZtYQEeEWQLWCnwdgZtYQEfl2eRRR4S8FYWbWCEMRtDkDjFDoQWAzs0bJpoE6AVTK8/fhBGBmTTPkE8FGcReQmZVDQJs/dUbwLCAzK4WhiFw/8IrILQAzKwVfCqJYSpEAbvvbbTywaeeRK5rZpBoKXw66mlsAk+yBTTv54XOv5B2GWelFhM8DqOIxgEnWJjE0lHcUZhbhC0FUcwtgkkmefmZWBIFPBKvmE8EmmST88W+Wv6EhXwqiWtHvB9DyhO9EZFYEbgGM5hbAJGtry/oezSxfQ/4/HMVjAJNMyGMAZgWQ3RDGLYBKhe8CkrRY0lZJA5JW1Hh+uqR70/MbJPWk8tmSHpH0hqTvVL1mmaSnJD0p6a8kHd+QPaqhTf7mYVYEngZaLEdMAJLagVuBi4CFwDJJC6uqXQ28FhHzgVuAm1P5fuArwO9VbbMD+Bbw2Yg4HXiS7PaRk8KDwGbF4KuBFks9LYCzgYGI2B4RB4B7gCVVdZYAd6blNcAiSYqINyPiUbJEUEnpMSPdFP5DwMvj3YkjkTwIbFYEQ24BFEo9CWAOsKNifWcqq1kn3UR+HzB7rA1GxEHgOuApsg/+hWQ3lh9F0jWS+iX1Dw4O1hHuaG2SB4HNCiB8KYhCyWUQWNI0sgRwJnASWRfQDbXqRsTqiOiNiN6urq7xvR8+EcysCLKrgVpR1JMAdgFzK9a7U1nNOql/fyaw9z22eQZARGyLrG/mPuBT9YX8/rXJs4DMisJXAy2OehLARmCBpHmSOoGlQF9VnT7gqrR8ObA+3rvTfRewUNLwV/oLgGfrD/v9ycYAJmvrZlavbAzAGaAoOo5UISIOSVoOrAPagTsiYoukm4D+iOgj67+/S9IA8CpZkgBA0otkg7ydkj4HXBgRz0j6KvB3kg4CLwFfaOieVZDwxeDMCiA7DyDvKGzYERMAQESsBdZWla2sWN4PXDHGa3vGKL8NuK3eQCeiTeIdnAHM8uY7ghVLOc4E9olgZoWQzQLKOwobVooEkE0DdQYwy5sTQLGUIgFIcgvArAB8NdBiKUcCwGcCmxXBkFsAhVKKBNAmfC0gswKIcAugSEqRAF7ff4htu9/IOwyz0nNXbLHUNQ201fW/9BoAQ0NBmychm+XGVwMtllK0AIa9vv9Q3iGYlZrvB1AspUgA09qzI27/oXdyjsSs3HxHsGIpRQL4+qWnA7D/oBOAWZ58NdBiKUUC+MC0bDf3H/TlIMzy5PsBFEs5EkBHO+AWgFnefEewYilFAji6M0sAbx1wAjDLmyfiFUcppoEePT3bzWd//itmTG/PORobr/knHMPRnaU4ZKesvW8e4OTZR+cdhiWl+G+aPaMTgJt+8EzOkdhEXHrmHP7o82fkHYaN0+7X9zP4+tus2/JK3qFYUooEMPe4o3ngunP55VsH8w7FxunffX8Tg2+8nXcYNgF7Xj+QdwhWpa4EIGkx8C2yO4J9LyL+oOr56cCfAp8kuxfw5yPiRUmzgTXAPwf+JCKWV7ymE/gOcB4wBNwYEQ9MeI/G8MmTj5usTVsT/LPuWb6vc4vz4G/xHDEBSGoHbiW7b+9OYKOkvoio7E+5GngtIuZLWgrcDHwe2A98BTgtPSrdCOyOiI9JagP8CW1japN4xxeSMWuoemYBnQ0MRMT2iDgA3AMsqaqzBLgzLa8BFklSRLwZEY+SJYJqXwS+DhARQxGxZ1x7YKXQ1uYLibU6N+CKp54EMAfYUbG+M5XVrBMRh4B9wOyxNihpVlr8mqTNku6X9OEx6l4jqV9S/+DgYB3h2lTUJjHkDNDS3IVXPHmdB9ABdAN/HxFnAf8AfKNWxYhYHRG9EdHb1dXVzBitQNrb5A+QFnfwHZ+JXzT1JIBdwNyK9e5UVrOOpA5gJtlg8Fj2Am8BD6b1+4Gz6ojFSkoS7/jzv6UdcguucOpJABuBBZLmpZk7S4G+qjp9wFVp+XJgfbzHPRjTc39BNgMIYBHgSfo2pnb5tp6tzi2A4jniLKCIOCRpObCObBroHRGxRdJNQH9E9AG3A3dJGgBeJUsSAEh6EfgQ0Cnpc8CFaQbR76fXfBMYBH6nkTtmU0ubxJM793H1n2zMOxQbp71vZucBfPyffDDnSGxYXecBRMRaYG1V2cqK5f3AFWO8tmeM8peAz9QbqJXb8BUkH9u2h/knHJNzNDZe53z0OG7715/MOwxLSnEmsLW+9tRZeepJM3nguk/lG4zZFFGKq4Fa6xu+i1SHLyVp1jBOANYS2tIHf2eHD1mzRvF/k7WE4RbAtHYfsmaN4v8mawntqednWru7gMwaxQnAWoJbAGaN5/8mawnvjgE4AZg1jP+brCW0vdsF5EPWrFH832QtoT1lgGkdHgMwaxQnAGsJw9/83QIwaxz/N1lLOKqzHfAYgFkj+b/JWsKMznTVEvcAmTWME4C1hA9Myw7VQ74pgFnDOAFYS+hoG04Avqa8WaM4AVhL8Pd+s8ZzArCW8MHp2RhAhweBzRqmrv8mSYslbZU0IGlFjeenS7o3Pb9BUk8qny3pEUlvSPrOGNvuk/T0hPbCprzfOv1Elp09l+vOOyXvUMymjCMmAEntwK3ARcBCYJmkhVXVrgZei4j5wC3Azal8P/AV4PfG2PalwBvjC93KZMb0Dr5+6ekcf8z0vEMxmzLqaQGcDQxExPaIOADcAyypqrMEuDMtrwEWSVJEvBkRj5IlghEkHQNcD6wad/RmZjZu9SSAOcCOivWdqaxmnYg4BOwDZh9hu18D/hB4670qSbpGUr+k/sHBwTrCNTOzeuQyoibpDOCUiHjoSHUjYnVE9EZEb1dX1+QHZ2ZWEvUkgF3A3Ir17lRWs46kDmAmsPc9tnku0CvpReBR4GOSflRfyGZm1gj1JICNwAJJ8yR1AkuBvqo6fcBVaflyYH1EjDl1OyK+GxEnRUQP8Gng+Yg47/0Gb2Zm49dxpAoRcUjScmAd0A7cERFbJN0E9EdEH3A7cJekAeBVsiQBQPqW/yGgU9LngAsj4pmG74mZmb0vR0wAABGxFlhbVbayYnk/cMUYr+05wrZfBE6rJw4zM2scn1ZpZlZSTgBmZiXlBGBmVlJOAGZmJVXXILCZmU2e26/q5WAONztyAjAzy9mif/rhXN7XXUBmZiXlBGBmVlJOAGZmJeUEYGZWUk4AZmYl5QRgZlZSTgBmZiXlBGBmVlJ6j/u2FI6kQeClcb78eGBPA8NphCLGBMWMq4gxQTHjKmJMUMy4yhLTyREx6p66LZUAJkJSf0T05h1HpSLGBMWMq4gxQTHjKmJMUMy4yh6Tu4DMzErKCcDMrKTKlABW5x1ADUWMCYoZVxFjgmLGVcSYoJhxlTqm0owBmJnZSGVqAZiZWQUnADOzkpryCUDSYklbJQ1IWtGE97tD0m5JT1eUHSfpryW9kH4em8ol6dspticlnVXxmqtS/RckXTXBmOZKekTSM5K2SPqPeccl6QOSfiLppymmr6byeZI2pPe+V1JnKp+e1gfS8z0V27ohlW+V9BvjjakqvnZJj0v6QRHikvSipKckPSGpP5Xlelyl7c2StEbSc5KelXRuzsfVr6Xf0fDjV5K+XJDf1X9Kx/rTku5O/wP5Hu8RMWUfQDuwDfgo0An8FFg4ye/5GeAs4OmKsv8BrEjLK4Cb0/JvAn8JCDgH2JDKjwO2p5/HpuVjJxDTicBZafmDwPPAwjzjSts+Ji1PAzak97oPWJrKbwOuS8tfAm5Ly0uBe9PywvR3nQ7MS3/v9gb8Ha8H/g/wg7Sea1zAi8DxVWW5Hldpm3cCv5uWO4FZRYgrbbcd+AVwct4xAXOAfwSOqjievpD7cTXRX3KRH8C5wLqK9RuAG5rwvj2MTABbgRPT8onA1rT8x8Cy6nrAMuCPK8pH1GtAfH8OXFCUuICjgc3AvyA7A7Kj+u8HrAPOTcsdqZ6q/6aV9SYQTzfwQ+B84AfpfXKNi9oJINe/HzCT7ENNRYqrYjsXAo8VISayBLCDLKF0pOPqN/I+rqZ6F9DwL33YzlTWbB+OiJ+n5V8AwzcAHSu+SYs7NSXPJPvGnWtcqZvlCWA38Ndk32Z+GRGHamz/3fdOz+8DZjc6puSbwH8FhtL67ALEFcD/lbRJ0jWpLO/jah4wCPyv1F32PUkzChDXsKXA3Wk515giYhfwDeBnwM/JjpNN5HxcTfUEUDiRpe1c5t5KOgZ4APhyRPwq77gi4p2IOIPsG/fZwMeb+f61SPptYHdEbMo7liqfjoizgIuAfy/pM5VP5nRcdZB1d343Is4E3iTrXsk7LlJf+iXA/dXP5RFTGnNYQpY0TwJmAIubGUMtUz0B7ALmVqx3p7Jme0XSiQDp5+5UPlZ8DY9b0jSyD///HREPFiUugIj4JfAIWRN4lqSOGtt/973T8zOBvZMQ068Dl0h6EbiHrBvoW3nHlb5BEhG7gYfIEmbef7+dwM6I2JDW15AlhLzjgixRbo6IV9J63jH9S+AfI2IwIg4CD5Ida7keV1M9AWwEFqSR9k6yJmFfDnH0AcOzCK4i64MfLr8yzUQ4B9iXmqnrgAslHZu+OVyYysZFkoDbgWcj4o+KEJekLkmz0vJRZGMSz5IlgsvHiGk41suB9embXB+wNM2amAcsAH4ynpgAIuKGiOiOiB6y42V9RPyrPOOSNEPSB4eXyX7vT5PzcRURvwB2SPq1VLQIeCbvuJJlHO7+GX7vPGP6GXCOpKPT/+Pw7yrX431Cgyyt8CAb5X+erH/5xia8391kfXwHyb4hXU3Wd/dD4AXgb4DjUl0Bt6bYngJ6K7bzRWAgPX5ngjF9mqzJ+yTwRHr8Zp5xAacDj6eYngZWpvKPpgN6gKz5Pj2VfyCtD6TnP1qxrRtTrFuBixr4tzyPw7OAcosrvfdP02PL8HGc93GVtncG0J/+jn9GNmMm7+N9Btm35ZkVZUX4XX0VeC4d73eRzeTJ9Xj3pSDMzEpqqncBmZnZGJwAzMxKygnAzKyknADMzErKCcDMrKScAMzMSsoJwMyspP4/FmHpub5Tc/UAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.01318556722253561\n",
      "0.3738261621211588\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD4CAYAAADhNOGaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWIUlEQVR4nO3df/BddZ3f8eeL/OKHCoFkkQVC4ppui2OLbAxSd5md5afbrewfMAtr3djVZtquM7U7nS6MM9jF3RltndV2hq6yyg51qmDBrhmXFiNq7bSzkKCIgEYisiQpQgRB11Uhybt/3PNNrt98vyHJvbnn3O95PmbufM/5nHPvec/33u/3dc/nc36kqpAk9ddxbRcgSWqXQSBJPWcQSFLPGQSS1HMGgST13OK2CzgaK1asqNWrV7ddhiRNlfvvv/97VbVydvtUBsHq1avZunVr22VI0lRJ8tdztds1JEk9ZxBIUs8ZBJLUcwaBJPWcQSBJPWcQSFLPGQSS1HNTeR6BpOn29A9/wm337WDP3n1tl9Ipixcdx29fsIoVL1s22e1OdGuSBPzlg0/yJ5u/BUDScjEdMXNrmFNOXMLvXLh6ots2CCRN3N59g/96X/93l/Hy45e0XE03PPe3L3DejZv3/24myTECSeo5g0CSOqSNuwcbBJLUcwaBJHVAaG/U3CCQ1Jp4yFAnGASS1CEtDBEYBJLUdwaBpIlr48iYzmuxl8wgkKSeMwgktcah4oNVC7tLBoEk9ZxBIGniqpVjY7qtzSNpxxIESa5Isi3J9iTXzbH8oiRfSbInyVWzlu1N8kDz2DSOeiRJh2/kq48mWQTcBFwK7AS2JNlUVY8MrfYE8Dbg38zxEj+uqvNGrUPS9PF8sm4Yx2Wo1wPbq+oxgCS3AVcC+4Ogqh5vlnkXCkmaQ5uZOI6uoTOBHUPzO5u2w3V8kq1J/irJb863UpKNzXpbd+/efZSlSuoCzyPoli4MFp9TVeuA3wY+lOQX5lqpqm6uqnVVtW7lypWTrVCSJmRaL0O9Czh7aP6spu2wVNWu5udjwJeA142hJknSYRpHEGwB1iZZk2QpcA1wWEf/JFmeZFkzvQJ4I0NjC5IWtjYvvdw1bV6JdeQgqKo9wDuBu4FvAJ+qqoeT3JjkzQBJXp9kJ3A18JEkDzdP/3vA1iRfA74IvG/W0UaSpGNsLDevr6q7gLtmtd0wNL2FQZfR7Of9X+C146hB0vRwrHh+bZxs14XBYklSiwwCSa3xhLIDpv08AknSFDMIJE2cJ5TNb1rPI5AkjWjqrz4qSZpeBoEkdUgbvWYGgST1nEEgaeK8Q9nB2rzchkEgST1nEEhqjSeUHczDRyVJE2cQSJo4Tyg7mOcRSJJaYxBIUod4GWpJveIdyrrBIJCknjMIJKnnDAJJ6hDPI5DUK55QdoCHj0qSWmMQSJq48oyyTjEIJKnnDAJJ6gAvQy2plxwr7gaDQNLEOUQwvzbGTwwCSeqAqT98NMkVSbYl2Z7kujmWX5TkK0n2JLlq1rINSR5tHhvGUY8k6fCNHARJFgE3AW8CzgWuTXLurNWeAN4GfGLWc08F3gNcAKwH3pNk+ag1SZIO3zj2CNYD26vqsap6AbgNuHJ4hap6vKoeBPbNeu7lwOaqeraqvg9sBq4YQ02SpkA8tfgg03qJiTOBHUPzO5u2Y/1cSVPKseKDtRmJUzNYnGRjkq1Jtu7evbvtciRpwRhHEOwCzh6aP6tpG+tzq+rmqlpXVetWrlx5VIVKUte1sbc0jiDYAqxNsibJUuAaYNNhPvdu4LIky5tB4suaNkk94AhBN4wcBFW1B3gng3/g3wA+VVUPJ7kxyZsBkrw+yU7gauAjSR5unvss8F4GYbIFuLFpk7SAeULZwdocOF88jhepqruAu2a13TA0vYVBt89cz70FuGUcdUiSjtzUDBZLUh9M6+GjkqQRefiopF7yfLJuMAgkTVx5SlmnGASS1CFthKRBIEkdMPWXoZako+FF57rBIJA0cZ5QNj8PH5UkTZxBIEkd0GY3mUEgST1nEEhSh0zrZagl6Yg4VtwtBoEk9ZxBIEk9ZxBIaoXnks2jhRMJDAJJk+cZZXNqKxwNAknqOYNAknrOIJCkDvE8Akm9ULR7a8auaut3YhBIUs8ZBJLUIV6GWpJ6rK0rkBoEklrh3cm6wyCQNHGeT9YtBoEkdUi1cADpWIIgyRVJtiXZnuS6OZYvS3J7s/zeJKub9tVJfpzkgebx4XHUI0nTqK3OssWjvkCSRcBNwKXATmBLkk1V9cjQam8Hvl9Vr05yDfB+4LeaZd+uqvNGrUOSdHTGsUewHtheVY9V1QvAbcCVs9a5Eri1mb4DuDiOFEm9VZQnlHXIOILgTGDH0PzOpm3OdapqD/A8cFqzbE2Sryb5X0l+ZQz1SNLUamMgfeSuoRE9CayqqmeS/BLwF0leU1U/mL1iko3ARoBVq1ZNuExJOvam+TLUu4Czh+bPatrmXCfJYuBk4Jmq+mlVPQNQVfcD3wb+zlwbqaqbq2pdVa1buXLlGMqWJMF4gmALsDbJmiRLgWuATbPW2QRsaKavAr5QVZVkZTPYTJJXAWuBx8ZQkyRNpTZOsRi5a6iq9iR5J3A3sAi4paoeTnIjsLWqNgEfAz6eZDvwLIOwALgIuDHJi8A+4J9X1bOj1iSp26q8VWWXjGWMoKruAu6a1XbD0PRPgKvneN6dwJ3jqEGSpl1aOpbKM4slqecMAknqEC9DLakXBncoc5DgIFN8+KgkaYoZBJLUcwaBJHXI1F6GWpKOhDemmVtboyYGgaR2OFbcGQaBJHXEi3v38dCu5ye+XYNAkjpiX8EJSxZNfLsGgSR1xLlnvKKV7RoEkibOO5TNbZrvRyBJGhMvMSFJPeYegSSplRvTGASSJs8Tyubk/Qgk9Yp3KJtbtTBIYBBIUke0FY5juVWlJGl0VfDt3T9i23d/OO86a3/uZRx33HgTwyCQpI74enN5ics/9OV51/nme6/g+OPGe/axQSBp4rxD2aF94Op/wIlL5/5nv2TR+Hv0DQJJ6pjLXnM6rzh+ycS252CxJHXMpPeVDAJJ6jmDQNLEtXGs/DTJhI8jNQgktcITyrrDIJCkjpnKMYIkVyTZlmR7kuvmWL4sye3N8nuTrB5adn3Tvi3J5eOoR5Km2aT3lkYOgiSLgJuANwHnAtcmOXfWam8Hvl9VrwY+CLy/ee65wDXAa4ArgP/cvJ4kaULGsUewHtheVY9V1QvAbcCVs9a5Eri1mb4DuDiD0ZArgduq6qdV9R1ge/N6khawqsl3f0yTSZ9sN44gOBPYMTS/s2mbc52q2gM8D5x2mM+VpF6Zuq6hSUmyMcnWJFt3797ddjmStGCMIwh2AWcPzZ/VtM25TpLFwMnAM4f5XACq6uaqWldV61auXDmGsiW15Ucv7OXFfZ5L0BXjCIItwNoka5IsZTD4u2nWOpuADc30VcAXanBGySbgmuaoojXAWuC+MdQkqcMe+X/P88KefW2XocbIF52rqj1J3gncDSwCbqmqh5PcCGytqk3Ax4CPJ9kOPMsgLGjW+xTwCLAH+L2q2jtqTZK6bflJS3nlK45vu4zOmvQYwViuPlpVdwF3zWq7YWj6J8DV8zz3j4E/HkcdkqZDFZz+imVtl9FZ03jUkCQdkWLy19PR/AwCSRNXVV5r6BA8fFTSgucJZYc2ldcakqQjUZRdQx1iEEiauCo4zhyYl/cjkLTg7avy5vWHYNeQpAWvCgcJOsQgkDRx5sChedSQpIWvvFVllxgEkiauKI4zCeblYLGkBW+fewSdYhBImrjyqKFOMQgkTdzgWkNtV6EZBoGkiSvvSdMpBoGkiStwsLhDxnI/AmmSnnjmb/mr7zzTdhkawdd2PMf61ae2XYYaBoGmzh/95SN87pGn2i5DI7rv8WfbLkGNXgXBT/fs5c++/Bh/81PvhjnNPvfIU7z2zJP5039yftul6Cj8+IW9XPrBL7ddhob0Kgge2vU8H/jct1h8XDjOSx9OpT17Bzc8P2nZIs5afmLL1ehoeNP67ulVEOzZOzhU4dbfXc8bX72i5Wp0NK7/9IN88r4dLFnkcQ7Taskiv4R1jX9NmjKDfyIGwfSauXzCOae5R9cVvdojmDl02e8j06uaA9B/8qLjPNPsE//sAn7+5BPaLkONXgWBpt/Xdz0PwOs99HCq/cNfsGu2S3q1f13uEky9vfsGb+L6NQaBNC69CgItHJ6UKo1Pr4KgmlECr3o4/XwPpfHpVRDMjBb7bXJ6zXTveRqIND4jBUGSU5NsTvJo83P5POttaNZ5NMmGofYvJdmW5IHm8XOj1KP+8IRAaXxG3SO4DrinqtYC9zTzPyPJqcB7gAuA9cB7ZgXGW6rqvObx9Ij1HJJjxQuH76EWora+34x6+OiVwK8207cCXwL+YNY6lwObq+pZgCSbgSuAT464bfXYpO/pKk3CbRsv5Dvf+5uJb3fUIDi9qp5spr8LnD7HOmcCO4bmdzZtM/48yV7gTuCPqo7dLStq/xiB/0Sm1cyAvz1DWojWrzm1lUOjXzIIknweeOUci949PFNVleRI/4m/pap2JXk5gyB4K/Bf5qljI7ARYNWqVUe4mdmvNdLT1QGGuTQ+LxkEVXXJfMuSPJXkjKp6MskZwFx9/Ls40H0EcBaDLiSqalfz84dJPsFgDGHOIKiqm4GbAdatW3dUew2F98dbKNwjkMZn1MHiTcDMUUAbgM/Msc7dwGVJljeDxJcBdydZnGQFQJIlwG8AD41YzyHt7xo6lhvRMXXg8FHfRWlcRg2C9wGXJnkUuKSZJ8m6JB8FaAaJ3wtsaR43Nm3LGATCg8ADDPYc/mzEeiRJR2ikweKqega4eI72rcA7huZvAW6Ztc6PgF8aZftHav/ho36ZnHruEUjj068zizX1ZsL8OD+50tj06s+pvPzoguG1hqTx6VUQzLBXYfr5Hkrj06sg8ODR6fePXnsGACefsKTlSqSFo193KPPw0an3rkvW8tYLz2HFy5a1XYq0YPRqj0DTL4khII1Zr4Jg/41p7GCWpP16FQQzjAFJOqBXQXDsrmsqSdOrV0Eww54hSTqgV0HgHoEkHaxfQdD89KxUSTqgV0EgSTpYr4Jg5lpDjhFI0gG9CgJJ0sF6FQSOFUvSwXoVBDPsGpKkA3oVBB4+KkkH61UQzHQOefioJB3QsyAYsGtIkg7oVRDYNSRJB+tVEMxwj0CSDuhVELhDIEkH61cQ7L9VpbsEkjSjV0EgSTpYr4LgwK0qWy5EkjqkV0EwwxyQpANGCoIkpybZnOTR5ufyedb7n0meS/LZWe1rktybZHuS25MsHaWel+Lho5J0sFH3CK4D7qmqtcA9zfxc/gPw1jna3w98sKpeDXwfePuI9RwWu4Yk6YBRg+BK4NZm+lbgN+daqaruAX443JYkwK8Bd7zU88fFHQJJOtioQXB6VT3ZTH8XOP0Innsa8FxV7WnmdwJnjljPIV1/54PNlLsEkjRj8UutkOTzwCvnWPTu4ZmqqiTH7Et3ko3ARoBVq1Yd1Wv81utX8ZM9eznntBPHWZokTbWXDIKqumS+ZUmeSnJGVT2Z5Azg6SPY9jPAKUkWN3sFZwG7DlHHzcDNAOvWrTuqwLnhH597NE+TpAVt1K6hTcCGZnoD8JnDfWINbiD8ReCqo3m+JGk8Rg2C9wGXJnkUuKSZJ8m6JB+dWSnJ/wb+G3Bxkp1JLm8W/QHw+0m2Mxgz+NiI9UiSjtBLdg0dSlU9A1w8R/tW4B1D878yz/MfA9aPUoMkaTS9PLNYknSAQSBJPWcQSFLPGQSS1HMGgST1XGoKL8mZZDfw1/MsXgF8b4LlHK4u1tXFmqCbdXWxJuhmXV2sCbpZ16RrOqeqVs5unMogOJQkW6tqXdt1zNbFurpYE3Szri7WBN2sq4s1QTfr6kpNdg1JUs8ZBJLUcwsxCG5uu4B5dLGuLtYE3ayrizVBN+vqYk3Qzbo6UdOCGyOQJB2ZhbhHIEk6AgaBJPXcggqCJFck2ZZke5LrjvG2bknydJKHhtpOTbI5yaPNz+VNe5L8p6auB5OcP/ScDc36jybZMNe2jqCms5N8MckjSR5O8q86UtfxSe5L8rWmrj9s2tckubfZ/u1Jljbty5r57c3y1UOvdX3Tvm3ocuaj1LYoyVeTfLZDNT2e5OtJHkiytWlr+z08JckdSb6Z5BtJLuxATb/Y/I5mHj9I8q4O1PWvm8/5Q0k+2Xz+W/9cHVJVLYgHsAj4NvAqYCnwNeDcY7i9i4DzgYeG2v49cF0zfR3w/mb614H/weBmyW8A7m3aTwUea34ub6aXj1DTGcD5zfTLgW8B53agrgAva6aXAPc22/sUcE3T/mHgXzTT/xL4cDN9DXB7M31u874uA9Y07/eiEd/H3wc+AXy2me9CTY8DK2a1tf0e3gq8o5leCpzSdk2z6lvE4L7p57RZF4P7rn8HOGHo8/S2LnyuDln3sXrhST+AC4G7h+avB64/xttczc8GwTbgjGb6DGBbM/0R4NrZ6wHXAh8Zav+Z9cZQ32eAS7tUF3Ai8BXgAgZnVC6e/f4BdwMXNtOLm/Uy+z0dXu8oazkLuAf4NeCzzTZaral5jcc5OAhaew+Bkxn8c0tXapqjxsuA/9N2XQyCYAeDUFncfK4u78Ln6lCPhdQ1NPMGzNjZtE3S6VX1ZDP9XeD0Znq+2o5Zzc0u5usYfPtuva6mC+YBBve13szgG85zNbhf9ext7N9+s/x5BnewG3ddHwL+LbCvmT+tAzUBFPC5JPcn2di0tfkergF2A3/edKN9NMlJLdc02zXAJ5vp1uqqql3AB4AngCcZfE7upxufq3ktpCDolBrEeCvH5iZ5GXAn8K6q+kEX6qqqvVV1HoNv4euBvzvpGoYl+Q3g6aq6v8065vHLVXU+8Cbg95JcNLywhfdwMYNu0D+tqtcBP2LQ5dJmTfs1/e1vZnA73J8x6bqa8YgrGYTnzwMnAVdMavtHayEFwS7g7KH5s5q2SXoqyRkAzc+nX6K2sdecZAmDEPivVfXprtQ1o6qeA77IYPf4lCQzt0sd3sb+7TfLTwaeGXNdbwTenORx4DYG3UP/seWagP3fKqmqp4H/ziA423wPdwI7q+reZv4OBsHQlc/Vm4CvVNVTzXybdV0CfKeqdlfVi8CnGXzWWv9cHcpCCoItwNpmdH4pg13FTROuYRMwc8TBBgZ99DPtv9MctfAG4Plm1/Vu4LIky5tvEpc1bUclSYCPAd+oqj/pUF0rk5zSTJ/AYNziGwwC4ap56pqp9yrgC803u03ANc2RFmuAtcB9R1NTVV1fVWdV1WoGn5UvVNVb2qwJIMlJSV4+M83gd/8QLb6HVfVdYEeSX2yaLgYeabOmWa7lQLfQzPbbqusJ4A1JTmz+Hmd+V61+rl7SsRp8aOPB4KiAbzHof373Md7WJxn0Ab7I4BvT2xn07d0DPAp8Hji1WTfATU1dXwfWDb3O7wLbm8c/HbGmX2awG/wg8EDz+PUO1PX3ga82dT0E3NC0v4rBh3s7g936ZU378c389mb5q4Ze691NvduAN43pvfxVDhw11GpNzfa/1jwenvkcd+A9PA/Y2ryHf8Hg6JpWa2pe7yQG36BPHmpr+3f1h8A3m8/6xxkc+dOJz/p8Dy8xIUk9t5C6hiRJR8EgkKSeMwgkqecMAknqOYNAknrOIJCknjMIJKnn/j+FvWImmMge7wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# TODO: experiment.score should be producing these\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "lh = my_experiment.loss_history\n",
    "\n",
    "task_losses = [r.task_loss for r in lh._recs if r.task_loss is not None]\n",
    "plt.plot(task_losses)\n",
    "plt.show()\n",
    "print(min(task_losses))\n",
    "print(lh.max_pct_recov)\n",
    "\n",
    "class_sep = [r.class_separation for r in lh._recs if r.class_separation is not None]\n",
    "plt.plot(class_sep)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "e9ddd7d6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LossRec(eidx=444601, task_loss=0.004459013696759939, task_val_loss=0.004254971165210009, pct_recov=0.8107209378624599, class_separation=-0.04687654566723043, user_data=CPNENStats(msg='en', epoch_type=<EpochType.EN: 0>, train_loss=0.004344584420323372, train_val_loss=0.004176819231361151, pred_loss=0.005402317736297846, pred_val_loss=0.002889303956180811))"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "my_experiment.loss_history.records[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c41c9bf",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}