3_hyperparameter_tuning.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CdhdXB_pPhcq"
      },
      "source": [
        "## Hyperparameter tuning with Optuna\n",
        "\n",
        "![optuna-logo.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAA+gAAADSCAMAAADudBqAAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAMAUExURQAAAK6urqqsrKqrq6qqqqqqqqqrq6qrq6qqqqqrq6mqqqmqqqmqqkR5qqisrKqqqqenp6qqqqqqqg5imQZfmqqrq6mrqxJsoCRvoaqqqgFZlqWlpaqrq6mrq6qrqwBQkgBOkABOkABLjQBOkKqrqwBRkqqrqwFZl6qrqwBIi6qrq6qrq6itraqqqqqqqgBZlwBLjgBKjQBNjgBIiwBZlwBLjgBLjqqqqqmpqQBcmQBIiwpzpwBZlwBXlaqqqqqqqgBIiwZtowBTkwBIjKqrq6qrqwBmnwBIiwBcmQBmnwdwpABIiwBRkqqrqwBSkgBdmgBIiwBZlwVwpABHi6qrqwBgmwNpoQBIjKqqqgBbmABWlQBhnKqrqwBJjABUkwBRkgBqoQ54qRJ5qkCUtzCKswNmnzKKshJ5qqqqqjyRtgBIi6msrKqrq0KWuQBooABcmQBpoABpoABpoQBpoQBTkw94qQBHi6qrq0iauwBfmwBTkwBqoQBIixF5qgBOkDaNtBB5qhJ5qkiau6qqqqqrqwBfmgBcmQBdmQBqogBUlAJtozGJshJ5qTOKsgBpoQBZmKqrq6qqqjyRtgBRkjKJsqmpqUiaugBQkTWMsw54qTOKsw54qUibugBpoQBRkjOKsw54qTSLszGJsqqrq0eaugBooEiaugBTkwBpoABpoBN5qwBPkABpoABooEaaujGJshZ6qz2StwBamDGJskiauwl2qABrooylsCSCrqqrqwBLjkmbuwBHiwBKjQBNjwBhnABQkQBOkABknQBkngBnoABMjwBPkABSkgBpoQBinABlngBJjABcmQBIjCOBrgBmngBRkQBjnQBfmyeDrwBIiwBamABooABnnwBemgBgmxt9rABUkwBfmiWCrx1/rS2GsQBdmi2HsSmEsBl8rCqFsDGJsyB/rR9/rRh7qy+IsQBTkyGArhx+rA94qgBqoRN6qiSCrgBSkxZ7qwBnniyFsBR7qwBUlABrogx3qTSLswBLjzKKswBZlzaNtAd3qEeZuk87D3gAAAC5dFJOUwANK8BXaZKKIZ/y6fYCQVEafzYIEaadDAXLFQfmcpYfPC1UXT0m4Bqt323ZOHklaahnM+hDfUqzE8fyRVJ00brMKraI++2V1Z5gIrrCM8rnk4FQ+YVLMXNdtIpZxEKUoKW2gFIZOqbjHCTCS0M6idJ/sPK/9FycVcP062qxa+1EmcWsFmS9qNzZ3naV1eTQ8C4pLtPTENfYa+mCpZjn5XaOxvdIhu3i/MjgNvn893e39V357/L19U7iq0Fy8wAAOmBJREFUeNrtnQd4FVX6/y+LVImoCSSEkkYIgYUEEoJAIqGFHnqXIkgRBaQqTUFRiqAorF3EwoLdXXf3eaIBggKJUoz5AQEDyuoCghRDLIj85T+nzjlnZm7LTHJj3u+zK5A7987Nnfnct573uFxlovC2KW3aDUxcMj29Z3xSR6SkpPie6X0TB3YbER0ZEeYCgUAVWGFR0e0WTI/vuN1E+/Zp/9t34MCBQ0npid1i2obCxwUCVTgFxcYMnJ6Ekf7IIPQzzjuh/VDH9IFtooLggwOBKgzkbUckJpkBbkI8xx3BfihpyYgo+PxAoMBXRJvEpI8Y5HuuXt2zZ8+OHTv26CL/uLpHph058siyZ2fHL4gOgY8RBApgtW2Xzg05gdqN0AEfXSWsfyTY9ezsIX3bhMOHCQIFpKK6xXPEGcy7rKXjLhp2xnr29BhIz4FAAeexj+gpQ05o3kuVo4v9SMd9j0b7VQ47yc1pqA9JjITPFQQKHAVF991OMdchVwFXpMC+gxp2YtYp6tnxIyBcB4ECQ+HdkgRbzhjXkc43SuJdt+wS6wT1IQti4RMGgcpdbZfoppxCLiKeayURd0o7Z52jjlnv2xY+ZRCoXJUyHYF5lWHOKdcRLy7OzT13Dv03F/+/+Nw57T/FHHdz1D8SrXr2dAjWQaByxLwnt+aYcgY5IRxDjVUki/ywuJjxTmk3mnUeq2dPB6sOApUj5ldVyqkdF/j+5RfyXyz8FwH4XGrbBdbNUV8SAZ84CFTmGpuOMSecU5c9n/jqxRzyX7jmbVq9IbNfV6x+mRtWL5qn405gJ3ZdQp1U2yjp2QOhsA4Cla1i+2KnXbTmlHIKOeV76+p+4wclpEbGxgXLLxAcFxuZmjD6sX6LMOvEi1dYV0P1jm3ggweByk6hC3TMqc+eTx123ZBv6jo6JtJzJ2tQREra+Mx5CHbBhceoM6u+D/fAI6VDrQ0EKiu16ShYc55+Ky7mkM/rN3pMrE9LTkMj0xI3kdy8iPqOqwx1Snp2N/j4QaCyUBTOwYmYE2NOKd+aOSjZv2A6KCqt61Zu1pEDL/jv3KjHQ/4dBHJe3bbrXju2vdxl1yhfND6mdMvOQlPHL0Ko55igzpJyYNRBIIfVNknGPF8w5ovGJdsx/i0sZfw86sDTUJ3470KkHgcXAgQqQ3POrLkWlo9Ptm8OVHBMv2KOOo/UddKzo+FSgEBOKTaeR+d7WWxOMM+MCbb7XOPmCaE6dd8PQE4OBHJabag5pyU1xDkx5uMcKXoFt1lNUJeMOgvUpwfDBQGB7FfYEsQ5L52z4PyX1QnONaylbtBJ36Nm36ElFgRywG3fLkbnudScb0h19rQpz3LU9yiB+pBIuCogkL2K2b6duu27BK89M9n5M0dvyhfcdylQh5QcCGSruunhOeEcee0bvMc8KCQianLysNQxY1JThyVPjooI8SFFn/a2dk5caFOT79D7DgLZqCUi59Scb/LKaQ8dnpqWMarDyoU7ZS1c2WF2RlrqcK9SaiGJgvsuNcSOgGsDAtmkkJ7b8bpz4rbnYM63pnl8Wviw+x6bpQH+maYvP+P6Ev1dB75Dr4TJnrN5kZuUQB1IB4HsVVQSS7czt72oKNHDcNaI3hmzPiOIf/nlJ0Z9iXGnvB9b+OigZE+mfTQL1MGmg0AOKHII4Zy47fnInK+OdPeEsOSMWTsJ4oTxTw0SaSesH1s4KsF9Y2vbTZx0yaZDnA4ClV4pKN1+VaiqnSsa5C4mH/PYwp3cjiOoPyf6gor+k/Eusl7QIc0t6+ORUTd675B7B4FKK72sxjh/1nq706Bhj+GYHFH+6ScUcg3uEqRCIvRXBjy17Qh2wnpBwaNj3KyLiX4bk75DqbKNhcsEAtnHOUq3F422PDZ25gBMOTHllHEM925FhHhCu27YiVUvWJhh3VAb8Swj/SOhytYRdloGgUrN+Uc658WLLKPzYR00ULEtx5RTxhHWR0zEeGewY9SZVT/+qHWBPpF576JNj4epkSBQKTlH8fleynlXC6KCEwYwzJHDrjnr1IxrTOeZSqcdW/ZPBdYR6rMsi/QjcEpOIb0vXCsQyF9F71PseTvz40IHLcSYE8o/R7acGnJK9UVFAu3MsEtm/VjB8eMrh1m8q1TdpsMkORCo1Bq7nQTojPOtKebWnGL+JTXmjHId8V81/WfV3AkZ961Z3FvT4jUzM2Zve/OIDruAOovVjx+dNdn8fbWdZ0I6pN5BIL/UdgjiXLfnq01zZEFpMuYlhZxyBvmqCfcNHm5ssAmLm7w4Y9sczDp14alVZ/770VHmK1EjNhlJ7wiLVkEgPxSRxBLuhPN+pq1rqQOO0Uw7wryEGHNGOYI8Y0wft6cJ7XTf3HeoWS8pYVadkn78pHnJPvxZ1DmD17Lt44Mo4JKBQD4ruKfAeU5ucaLZQV06HOPB+ec4AUeMOfHWf527uI9X5woZM/vdPIL6FxLqx48fHWCagA/dwNay7TtwAMJ0EMhf9eUNcai/vXicmdeeUYCcbOy1o9gcWfM8jvm2xb6Mag3tPZejjjLw3KgfPdkryIp0JUyPhKsGAvmmgfuEAD0318yDTh4gmXPmtGPMt8zs4vMp+2TM0VH/5BOUlMOkHx1glpQLedZAek+4bCCQT2qzT0y4m5XVgnoVUM6ZOdd99lW9/RvvHrzmZYo6c99x9v3kfrPvmZBNPCEHGzuAQP4o8oAQoOfmmiw9nzwAc47N+RfInOvWfFsn/08ctHgLysATo84C9aNHTz5q0qgT+yBOyInOO2zWBAJ5r5Ak1vmKE+4m9nzQccQ5dttJrp1j/kSn0p07aM27yKhLpB8/un+ACcORJCEnkA4NciCQ91rCA3SUcDeuYgntUMDD88+ZOceYvzy49GcPncAidS1QZzZ9//4xxiOj9TCdmvQYuHggkJdqd0AP0PNzxxvd9oUFBSw816NzDfN31tjzBoavOkLcdxqoFxQcNQ/U27F5kXx1C+zqAAJ5G6DvEzpluhoeTzhacOwYS8OVCJzPDrftPdxH3XdGOvLe9/cyHjdedd7bwfUDgbxRWE+WcUecZxqK2KOPE7+du+0U8/8MtvNdDH9TDNSp9z7K+GY30Mw7a5vpCPusgkDeaCA16Liy9qChg3zUUe63S5zPtntJuBapF5bopBdozrsJ6VG0xsZN+kC4giCQZ6UcQAb9KnXc1f7T4A7HCeeCPcdZuN72v5MxR2hF3a1Nb7OLmXRC+hBY3AICeVRoPHfcNdDVAnr4yqOkroY5J+E5XrrSx4n30uXl3btLRO/d1KYnMucdTDoI5Kvjjg26mnCPGyByXsI5n+DQmwneJnnvmPTH1INCNrF8HETpIJB3ijzESuga588qtaoIxjmLz2l47kVRLeyutzrNXzoRa+n8wW/18bIMNoF571q4UFBwXAN9v6GsH71LzsdB4h0E8qB0MRMXabDnR3XOdb/dQ7Y9pNPSZcunfiXpu++mLn9xYqe7PL+hDOy9484ZGqbvTzNx3sUSWxLU0kEgtxohZuLamcTnBbi//dNPP+eczxnuLuIfvHm5xvX581/LOn8e0/7d2s2DPSXrZ2Lv/RNi07HzflidGxn+oJx4h61bQCB3Cu94QC+h91Oc71knj1LOP/kUtbcTzt+0TsPdtfTODzTINa7PfH3mzJnTTNrfCe2Y9Q/unB/qiXTUOSOE6f9UZ1q1kU16OlxJEMiNBooRurIpwqMnhQCd5+FWWTXDhc2/8ztKucb2D5oOHjyI/4v+jnFnrH//wYtveUM6anvHoO+fpXbxZMqJd9i5BQSyVtQhFqEbHfde+8VEHLPnqyyMcZ/NxJYjO44RR/oRi/4DwY5Zx6hfXjs/yF2cXkJJpyb9sNoMO5YMkGOJ9wVl9Yk90vSGVjXqN2/fvnP75rWqtxp5081wF4HKUK1fr3ZLk3rN0f1Xr/otK+6t6dWzEgWDvkF+aNB+ZNBpgK5zbr538lvUmHPKfzyh6f+YtL8T3jHsCPWvNNSnLrV+Y7NpOV0nXV2lliiZ9I7hZfAZP9WiXsvuWaqaNW/c36kzrm9Sw72G3tJ4Rd1bbfyyGVm/RmlUfWgVP09cs7r8SvX/4uEJVYbKz6hV15YP4P5ayvto4ekZraSLVL3GbU7dDUFNb+ncyHj/tW/1V0/PHHuIr1rLyZFHuKfu369X0L8oQctSL1789U3TivXwO7/7TsNcM+aIcs2GC4zrOkGMO0NdI/3y8/Mtf6cncEKOheka6P9U+t+iaJROZ804no5rPbRllqWaVXeG9ZFZXql7s85D6/7NljO2zyqd6jzi54lfV1/pfk/fDOp3bhNbPoB6hl/pJg/PaKYc/5QzN2DT+j0sP/VGTdzff0uwQb+KPXd55mvsP5FB5447DtB//XWOWR4uTsacQv4t0v+w8F+/ZbBLqP+81iqD32cOT8hRk652yI3XTLqejnN4AMW9d3i6w2+/wYHT3uADYt2rVrPBljQvJei1/X0PN6mv5MlA31xHeUJ1Wz7yWoZfqbOPoNd05Cu/paf7r65bg65H6FImLmzWfpqJkwJ0s5GNE7/XvHbktP+APXYCOeL7J1GYd8o68uC1YP08CtV/vrTMYtjcsN24b0Y36YcTzE36Aecb3v96uzf3eLO65Qo6UvP+ALrtoHt6J2UAet1mXnz0Le+1enpf4rnjCL2bkogTMnGcc5M+mclTv0fm/OvTp3+gmGNDrqH9DdJ/kfDfvkGwE8OOUT99BifgNaM+dbJV6p2bdLyO7fCTcUaTrtfSnfPdq9Tz9i6/Y1o5g669hfUAut2g31HOoD/S2dvvefMESeQh3uWeM0+CKGY/zcTplbWLpn2vyy4Tt13A/H+YcQT4WV2U95+oXddRR6Rfmmj++83Fzvtnlpn3SAl0x3z3pj18uM9HljfoWVm1GgLo9oLu4ao6Dfo9Pnz6Tc1T7rrnLpXWIhYygy467hMML9BlKuJci8455phygvgpWYR2YtcJ6gLpy01bWLu8oznvn4jOe6R8QFc6EdbRvHs132706uUOelaPmwB0e0FvWZ6gt/Lp869mfIGo7APcc5cN+ihk0At0g445f8JQ9l76MwnPsTnHmP+PYU7g/p1JYJ2hjow6zsl9951G+lTTbrvFhUIxfb9Gegf58RTJpDuzueotvt7p9cod9KysxgC6raBntSg/0Gv4eAFaGV5hoOC5S9svJRw2GPSLv75j2Idl2c+Xidt+8ODBExRzTrlG92+iKOwYdc2DR0Ydu++Y9O9/vnRpspXzTkpsxHc/rMyFTRcHwjrSM9PK91u9XvmDXop7HkA3LV9VKS/Qh5b6Wz4kSffc3xZbyeOexAadldCp475YeXrwcp1zYs6RNWeUI7b/nygddsw6DtWp+05s+gWz4fCTWT6OmfRZ8uMjRJPuxPZMI/2512uUP+j+3/QAupd2smxAr+bHJVAaENpkHyBdcbt27ZXGTfQiBn2nZNDVAP2utTrn3JxjzBHlDO/XNr4/5YVXXnnuuVdeeHzKq+s01hnpP2HUJdLN1r5m8HwcNelyiS3ibdF3j7Kd8/X+3ewjyx90t3cmgO4z6LUfKRfQ/+pXz1Jr6TWmC6k4McmVfHi/0CtDDfrLSudrn+d/pmU1yjk258hgc1O+bspzM+TV50F3T3p8I0UdJeAFm/69hU2PmyN0zSDQFZOeuEOYP2F/ga2Bnzd7zfIHPeseAN1G0LOGlgvot/t1DaqKL9E2m7e/7s0UH+igROjYcR9s4Pzyd98J9pyac27NX33laYu3ftdzr2LUtUhds+k4Jfc1Jv3SpX+bdMndh6J0kngnJl1emR4j+u62B+mt/L3bmwcA6I1uA9BtBL1763IA/RY/L4KYeh+YfYiPihNNYYxg0NliFtVxJ5x/ZeScYL7uhbvdvv27H3/tFHXfifeOSddM+vPGClnIyyVilH748KPyww8KK1vsDtKn1Slrc2on6Fn1AXQbQXfTSe8Y6K27+3kRbtTXPQQnkRAdGfR5Al9BKw/rqTgWoc+RW9JCpl66/D3m/AfEOXPbKeavPuT5Nwh9gbrvjPTzX+Ewfbm5Sf9U6HhXaumJpOEdgz7E5hmRNfy/3asGAOhZTQF0G0HP6l/moNf3+yrcwl8jOpuE6MigJ0qlNea54xo65vyi0hK3HHN+nnL+rWTOX53h3e8QOoWRfuLgwR+0MP0rHKZvtorScS0dg97L4LvzAluKrZzfbGnQezRoXqtJ9frN77jR8pN+PQBAb+/PGctt9VrAg968rEGfZmHQe7QfuuKee5veW3dFjc4W96heDkzUu2VyxE6TWdyg0whdA32bfP4XL13CCXeN8x+x365zvu4h73+Nh9Yh0lGVjZCOnXeT1HtG4RdSOu5JKS8Y9/YOPUgfURYR+h2N+/Me06CnVnR2NEo3gF5jZAtJjW+pX/VGn22QG7XoXM9MtdST1KlXy+y45vWr/FlBt/SQnALd/AZs/7rY4lzl/qqmR61g3ndS9iHiue/Y9WCIYtAF0HGELsM38QJz3A8eZPE55fwVn36PsClnSUH9BE29I5P+vGGwxXC9492swtZXCNITbQXdNOPZ3rDYeJr5nVHTGdBvNTkoqP8tzRxvx+3sSwvJnxN0q+WqLR0C3WxhagPjOI6mzdzEjtE4505C9PFSyl3w3GmEPlt62U4XLuEKOknEffst41zz2u/y9Td5BZH+Pxam48z7hWWGo+bydBwBXd7stZ0QpKfbeePdauaXmibZ+jfytWnSXtDxgWYrbxo1tO/jUA3HjZUPdKt4zCHQb/L6q9ssr0IN0gIhRBdmNCUfVj33vLyLUnk75HnNoKOMO0/EIc6ROX/Bj99lErbpiPTTzHm/Yqim9y4kS1uO4aaZw4cPSyvP2RI229e1mDhOjSwuYcMGNgXI/oPuCmruXKYAQHe7XNUh0Gt433NZy8rQBMUTz/3qjl273hZy1Y8dloroeDmLbNBf1Aw6ycSRAJ377Q/59cs8dJYl5E6fOU/ycYbMe8gWlHf/knbHHVZ899AkHXRbe+M6e+o4EiMlo5fV47ayBd3lqu5TlweA7jvoWTeUJegNfEj8WB0bma0X14Rh7hH/PMzaXz/7ktXWpNUm869cQhH6edIRRxNxmt++7m4/f5tJJCH3o56Pu2KYIzcBrVb97DOed5dnSvXboQfpNi5gq3Kjpx5iD35+07IG3cSDuwNAtxX028sQ9GnGyqX1DND+hoOb4cWm7TDopC1O2Ngs7bDeLcNScXMl8zmVGPSvz6CMO03EafZ8o/87pT+HnfcTtG0GRelr1UPGCKV0nHeXPPRuQhesjWn3pr454zVsazb3H/TbbsxyLmMGoFtmXpwB/R43xXGjjCV3fKf0zdYb3VOUVJweomODLi0M3XwBpdy/4gadBuivhpXiF3r8v55Mevgc1gZLlrBdk9pgY4RsXDf77ruRvplow0hSe2aS+gS6q7HhPa8H0G0FvVnDMgPdkCOq426o93rTMCOko97o/qBuHqOuSd0ymueel7dKHDfR5coFXEMnBh1l4rDjvrE0nLuCNmKT/iPuj8OgG0z63ELWM0Py7tLuqrFCu7uN9bUaPja71XOkOc430G+r7VQrLoDuxk9zBvT6vs05qGqWn0nJZlV0aUGL5rnLIbrmud8nZeKu0Bq6ZNDXuSur3T3pvZcefuCBBx5+6b1JFsc99F+Sj/vBMvF+H+2ZYf3u0qCZ0Ad10Kfbd9/V89EVv9/w5V/2oBvf9AoAvXSgd/cix3q7I6Df4dvi51Zm3wvtsg9x0Mcpnvt+KUS/2Ecx6D/LBh057k9bnnzGS898LOqB90xZfxzl41DinZn0F5UDJrsN0vuhyXEE9HQHk+63esidqLddj7+VPegtHEkUVGbQOzfzXOFyBvSWFpVxbz++zjhE15Pu+sq1COy5nzx+/Ji+El2qrS27oqfimEH//bd/WJw59L2/f2zQ9ZdMUL/rtZ9I18yZ8yQd92+lHo4LbJ/ooF9LFh9NJCMiEejxYbbddw18XayhXu7uNcse9Ka+5G8AdC9Ab6xayjrTPN4qtoBuqPp48BANSaLbXa6weCHpHqlnta7RXByeLfM5DtF7Cy8V92/iubMa+k+4hP64+XnD3vjYXNffMx78Ak+8a777JZN0HA7S9Tkz19LEB4XeOBs7ZtQv1AaentDciUSYj6C3BtBtBr1GUB2PSVZHQDd4iFV9vGGbofGvQl+c3mY27poeouN9FY/k/UcEZ80VlHPXa2s4QrdIxE3618dWuv5AqIlJ//b//o/67pdNfHeysEXPxvUySbsfsHW7lobNfO10q27HkpJSgv5IbQDdXtCbG7Oy68sEdMN3di0fL1AP1OiezbdoeVY/MpOCLoTokue+VgjRT5DVqad+M22IC3v4Yze6/neD+z4FmXTuu1+48rzyXbCYgr6TtsxI2bgUoQnWtta4ho18/JyN00BuLXvQb+sBoNsLelVjKaOep7SZLaA/5esKJXV9cW3cLsNB1/viUFvcYdouw/pfxdGvk/8gIToqouNUHDLoU0wz7X//2C3o159Rbfokko77ga1hU/PuyYVykP6kuOED2YONZOPaOga6x7K4Ie35VwC94oPewOQbvKmHj8YW0P/ibZ+7VcWlNlqLznJxO4Sla8nXCOh0uAzOxYmz3Cci0C8rqTizjPuMjz3o+vWHlacEryPpOLyGDYGubNLUB+2WzltmDh++Jo2nfltfqRrpGOge76LGAPqfEPSWLleVRh6iuM5lArqnS1nfCHpPoYyub8WUdo33xbGk+xPiCy2/cuWSDPqpU4/7wzki/Tlz350uVr1w5U4lg7+lhLbMkHUtUto9VJgbN7b8LDqA/mcEHbXCtfDQhhSooIcmZbPBkGJ1bZzYF4dXtORliEYVGfRLtFsGD5DSPHeTpSxPf+wN6H9XnPfnSN4dLVbFoD+vbMb2BF7XQrNxmuchLK11BcXvANABdGdAR90QhsRsVffRcaCAHpUtgK6v9+pAQD/K+uKUEH2wSYhuEqGHPqNS/fcHHn7GSLpSZHtLDNIvXbhwRdmLba6SdpfqaxtQ2p3E6CmVG3TIutsNOm4wX+F+uWrzAAV9bLawpEU3gauF6hoqo2ugi3PWNxPQ2Up0HKJPMp7uJQXph3EU//QDKugPKN8PH/7EK+moZeYPJRs3gTfBmnS7Z2LQ94FFN3ZNAOilBJ1QqzZDtXSbBgsU0GOyhTlSPE0d/qRURkegbxHnt91Jc3Hnv+ZV9HXGxamTFM559e0NlXTF69/IsnG03X2p/HAGbYI1HQXbj9TXAHTjsngAvbSg48ZTw0KGam6eETCgtxMtOi88R10zgC4uRQ+aegX3xem5OFPPXXbS/yUk5R9WQFe8gfe/cZt2n7mbgX4Up90fswA9snKDPhJAtxv0W01/e2m5av0ABX2gCDpvJUu+JvfLFO4+MkF4mfB/I9B/ZsPiULvMKWOX+z9kuz3JMhlvCNJZ2v0MBX2ZOeg7KeijLEBvW7lBb6Ie3hhALyXoN+Ef3uPug20SoKAniqDzgXHDrh1WQRf3bRj+xx98LfqPJ0ijuzHnLofiD1s/dv36G/IzH6ft7hro35k0wd63m26UTjtmuqqg02b3qMoNektYpmo36PeYZ9bF5arVAxT0vmKrOwc9RgCdjpcRh8t0+kPvdKegrzO0ud8tG/QZ1mk6S9BZt/udJqALrXGPWoAeW6lBr5sFgyfsBp0OCmzqZu5mjQAFPZ2CflUD/W2+aiXBCLqY+UbVtQt0jSpduva+4VzPSZw/Iz/4hlvXXbfovoOeyUHvGFepQTfOAu0PoJcS9JEWqfU6Os5DAxT0ngLoukVPuKZPhqSgi9W1+Rh0yaIbc3EvufHc1WzcJDVG1wvpPxtBxzG6FegbKOiHspNCKzPorRyaOV2pQWf59fXWMwFbBSjo8SLoEdagv9PHPegveAjR33D34PWn1ay7W9AzFNDFZFzYJg66fRsnV0DQ7zdOOGvgAtBLCXpjK5Cy/mJ15QMF9CQR9Fhr0OeEWIFOXHfjTmtyce0l6TG5Nfa64te7Niquu5KMm6C47iLooQ9y0NMrMegrTEYZVgfQSws6H8ZlaEbiy1VbBCjoHUXQ21rH6FuCDTG6ZNHVhSmuIHl56gNu3Prr8reAK/hDJRm3TH58tgK62DAT8TaP0ZdUWtBvM93X/AYAvbSgD7VMrvNrsaJMQG9S5S9u9JThBtBBx1n3sWZZd1JeezlYyrrr5bWDFqArFv1jsfymrGm7rmyj/vRZDjpZvqY0zGxTymvCSEtX210c9AWVFPT+9U230q59G4BeWtD1ZeA317ZYrjqyTED3VQLo0qKWYYaGmZfF8tlbf7Dl6KcZ6K94iNHFID1UmUWhtLq7/nGWLV87b9YCG/am0jAzSHgwZRdvmGn3pwf9/iqtJT11a93GtZpZXOvmLgC9tKAL113Nrmfda54cCUjQ+TJVtTOuRLboemccGyR19nEPWfePP+atc4aZM4pBdz3+X9oCy0CX92QPf1cBXdxnMUYHvc2fHnSfVBdALzXowggxwwQKulz1nkAGnS1q4buVGXvdt4i1qqDn/7hCt2Nive7G8to/1OWob+B9XoLfU3/+kvrMjQx0tqhFXqY6fLeyqEXclKmbDvpYAF0e+Auglxb0eu4uMcmBvB6goCeJFp0Hu9LqNbxM9V1pdjJZvcaGPaPVa68a3lyIYfbrvx5+442HDT81zIx7Gu2eLA58VqZDjtFA//RTYZmq2NOeqMfoEQC6I/2vlRh0MfxpaD4DvGmAgh4vgq43ja9WB08USlZ1mTR4Am2Y/FqoR9/dQqrj7nrhLN2shQyeuKSW0dfsltejPylW/jJ3sfXo9vXL/BlAb+AC0EsPemfxsRWmy1X7By7o+oQZfes1PmGGj5ISO+NwIV0ZJWUc9Xy3V5wb51VsZHsn0wkzVzar1bVCaZTUSjF+f3AHtugHDtm59dqfAPR7AXQbQJfHRhkmUASZMBkooKeLoD/IbeM4w3BIKSHW5Q+2lSpfp2psjXO94QXnxtWtD52l+yyeZv0yci7O9YQwBfak0i8TRUJ0BPoCAN2RZpnKDPod0oOG7kPUT9M6QEGfni1Mgc3hCzuFKbB03HNv6cxr+fK103QK7Ksm7+8BP+y5awoCXZw78W95i4c+hbvlKbCjjUl3BHobAN3iBgXQ/QVdCYDUzRpu/Jtx96RAAX1JNt3AYc+OHbtyovX6mr6BA6mvzZTOzIfG8SD9rMlUd+NwSGVUpMlznj5FQ/QfWNJ9uXzA4CN0A4djdK77GDnpzjZwSAHQ+QiURwB0W0BXShevmyxXVfdyCRTQF2QLO7Xk8B6TWFxfE0GXNmTCK9LJZqpkd3QtSDfx3V2hbm36w2ZbtWkGXd6S6cIa+YCZLBfHtmQSk+79drEtmWzcYrGig35jTReAbgvoLZWH1QkUdaYZ6uuBAno3CfRE/tKzWMfMMVJIL5T2b0BT4y4oOzisCzZ7i9ap92dmmB0/49RZXkWnIXoX+YhtqIxO9l7DSXcxFxfy4C62yWJPF4BO786bXRUO9HsNTYAenjCtXEBvasyFBDULTNDboN0IebO7vstiL6WQXviOPMZhMx8PiX1383Z3TK65+/7MJPNf6X1i0E/o7TKK5x737m60bTKqrpFtk8XRkGP3ctAXlCvorQIH9OZBrooHuoGgkR6e0DqrLECXxkAiqRMoNK5bBiboKRR0mo3jC1Xp/ujHj/O0e7J06uGskn6G5903WrzLSQ8YO2dmWBz73KmzUnFNA12Z9dz7CFuNfqzgqLp9QzsN9KukX6aNnaCr/eO+76bav5xA7z7S5fozgN7CwxNuLRPQG6m/Z3/jBIoGgQl622zSMUNBj9aDdH1FOlnWcp987julfZPxZi2vWL3Pu997+O/6Xi1vPBRsdeBdH57FqThh57XnQ+RDJrAGWLZ2TQzR++7agUBHubi2dt53LS2WKlnKsIpxffmA3qSKq0KCbgDI0/ahr7sZ42Yf6Mbf0zCBonW9wAQ9pCMrpF9FoOsrPjsclrrdS3bPlc/d6Qprd2c9M/997W437/WuGQ9NmjRpxt1h7n6h92WDjjz3zUp6b4veF4dBl9pl3ma5uEPxtt53t/vaZ6auB+5esxxA71HjZperYoLe2te1d9V8Hc3gF+jGaVw1DV+ttcoC9EZVG7jTHT0MoKOhcaQJFtXX9m7grz1I2GbxE+S7bwkxMenf6+k40wmRvukFxDkqoh9kOyxeutBFLa6REJ3vvCYuRo/ey0FPdPTGrvOIj98Mtsxr8wn0G+vVbehyVVTQb+7u45qcJr76+n6BXtt43dWpr7V7BOaEGb2QrgTpk4WWGTJkZpiCHM67k/3XkElH+zK9UqpfZ9KpU7y2xlJxyhQp1wQ9RCe5OPFdjd/L1661sfW+M0xsaer+eMOkoWausgO9R7OqTVrcGuRyUI6DblgE6gmYBr6m6f0CvY7RRTJsZRmgo6RofY01webogMySx8Z9sTvDZTTpl6lJp1uqmnS8e68Zv2POiUH/mqTi1Npa+BxSXOMh+gAhFAjdtIsn3WNtve+q+/hBj3RkTYkB9BWtm6r66/rWf3M5LsdBN4K7wjdX38PmFn6CbhaBDa0goMfIaXfd5R0kBOl4XcubyosN56V00jSDSHcbprvX3R+eQk1xQoR++dKFZcpBi48oIbo4Ly5lrwY6aYBNt/e2q+YjuAYPoJYjoN/qKic5D7rhE3Sf/2zlDGDebJlodD4CE3S0QTrpdv9ox65dOfq6li5CkE4KbMruxa7NQpSOTTraVPUufzlfxxx3HKGfxwb9A/XVth0pLJFCdFPP/cChbvbedoacbtbr7g5/KsuR1FBlAt1oJ90WKA3bT7V0OQJ6a5NjGlcM0EOTGOh4EKxeYHONkoP0EoPvHraWVNhoLb1UpD/94SnmuAsGXamhuzphgy4sXZPb4vR2mbH23naGBktlwaKHj9mm3ZAqE+g3+DTzzjjdur0zoD9lckxQywoBOtp9TZgmlTOeH5wqrmtBzXFzQlTuuEk/ffBH6ryfXeeP9/7Qb5jzn76ly1lwm/ul5ephs3nOnYbo4lzImByei4u3OxV1e5bplAFz3ePQ5a5MoBuDbjfptUd6ZJmtGHUA9L949zUTkKC3yxaXpAu+e9BKHqTTvPti9fUmYpOuO++aTT976jXfM3LPIc5JgE5Lazjl/pZyWJcjuwvlnLuYc+uaw9eoDrT7vqtuvIKWTa01ezg036UygW7y1VqntdWxnY1X5yZnQF/v5XsNRNDFJtgdu3JyYsR0HO2CJT0zysIWnHmn7XFkERupsZ095WOVLWzK74hzEqD/yB33S2vUAzOOsHFxxKDvF/vco5BBp7m4FLtvO+PWpFl1LEpsNU2SMzUAdBu+Wm+0IL298dBGDZ0B3TxRcH+FAD18iLiAbW+OPjgu/EmhOQ6n48aoLxgylZXYCOk4TD976tUQX9z2Dwnn2J4jxx1l4jSDfqd6YJ93j/CZE8RzF/vvR+fwED0+zO7b7m83mlzDxqY0mu2e0BRAL33+Myuru9lmMzXN7Gl9V1mCbphAEZCg4yBdKLDpPTOu0cSkF/BS+lzDK76FTDoL05FJ/+kbDfVTrz3n7e8QOkVz27X4nHCOW2WI4/58uKlBJ5sxEc+9g/g6m1BxjaxoGWj/rV3f7CLeblgl3dT0kts0arlSgR5kuhHFHWrAdHN1U7zucQj0W73/Vgo80NtJLTN79ekTyKTzhS24lH5ksOElB9PMOw7TcUIOZeROndroVaQe9sJr1G3/hthzGqBfvnRhssGgv6MZdD5zAnnuooPRJkcvrkXaf2vfa34ZG1W/vzX1EhvWvKeGRf61FYDuuywaUVoOfX0aTbX+rX+LzuYH2TbEvpaXiZnOFQH0SJKN4+m41WFylM5L6Z+X7DZG6a75mvMuko7idM2mnzr16gyP1vyVD3+jnP8kco5K6EsNR0/II6k44rkrBt21QQc93Yl7u4F1V3mDqp07V21g3TdR52YA3XfV7G49GqtB586d72hZx/KAFmUNetOKADoZ+ax3wQql9NABwohIVGFTZkRiLSU1tvM09Y5z79ion9r4j2A3b+bpx19DmCO3/RvqtyPOSYA+0XD45DxhPQsuoosGPSYnh7bF7Tswwol7+37/1wjWcAHo9qTjvFSjKk6BbplsqVcRQF8gBum7cnL66U9IIKV0MmcGmfQ3Q01Jx2E6mv384wkSqP+XoP7alEmm+ygEzXhh428Ec+S2C/YcVdAvX1pmfMpcobZmNOj9cM4dhegHhsSVyc3t/WrR2wB0m026BzV2lTno/SsC6CnZcnNcvpDM7iCn4wqVabDUe6cJOWTTSZWNG/VTv7/26guTxFXoQXc99MqUdRrlFHNsznH9nHH+86UXjefonUdT7jxCF7tfo3P0nPsCZ27upv7edrb5kZUMdJMGdu9k4zZztbwuz9evAKCTLli9OU6osLmSeZRO+2DlLVuoOomkM6POrPrvv//+24cb35/yuKYp77+qMU4pZ5gjc65xjvLtxG834TzkZSHljkEX921wZWqe+w6661qkQ3d3Df9uu84uAN32vIhbNXUOdOsNb1pXANAF352U0vOFOUy9xIlSqJa+zeyFu0zFpJ/H9XQUqH9LC23ErGPaEfDkD/Ijgjkx5z8izkl8fvnSZpMTZOSRhejUoGuee5Qcoe9lbXF9Hbu97/DLcb8ZQPdX6/1y3m9xlQfoVnYgoEAnvrveMyMsVnWFDNgv9sF+sVsdHkcUvJyTTtx34r9jq05ZF6X9UMD8hM75ZbM8nMs1LO+ImHLXDLq4P0vQs4LnHuPY7T2thx+3nY2bnlU60P0ah9ncVT6g31w78EHH86TEdFy+4P6m6ol3tFq1ZPeRTqavPZGSjo06QR1bdY11BPtZDDz9i/ajb77BTjvy2inmtK4232XpuPOF6Cf3rxTz+WmEc9wtE+/gaJX+vluYkS4AvRRq7PMHXtVVTqAbJ/8GIOjt9FL6DjVKd/XaL0TpyHl/03xD4remItK/On8GG/UfKer/+wnBjmhHvOM/0L+JMUfW/MRBzDnue73889QuZi894QgprfE9VPenCo+GP4hra8RzH+HkDf7X7uXIeWUE3WfSqwaVG+jmEygCC/TYbHFynGbSc4VlIcErMei8xFaobM+ke9DL8FI2zaifRkYdo45Z12BHuCPi0R+a/keNOffav/4Kc/6iaZf6GoPjLg6WcY3Lz2GzZfbFBzt6h7du5Mtd1/11F4BeSo30ifN6Np/dF9DNv5QCC3Q8IlJMx+VkCs+azE06zsdpzvtMi9d/ay026sh9J6hT1jHtur4lkGPKNcxJdI7ScPNNX7VTnu64ky73AeKimcjcHL0rboTDt3jD9j6Ueaa5APTSZ+R8+G5t7CpP0E0nUAQY6NHZ8kYOOfniGNUEnHgXnPcji63OsPQDZNQJ6siBp6xrtGPev0V/EMhPqJj/fKf5mrc+c47sxk3uemlNGkibmY+bZTDoSSGO3+Qj65RD9rcSg+4KauLtF+tTrnIF3ThaMPBAd6WLbbCowrYoRA7TqUmnbTNHhlmdInTz95dRpE5Q/+GgCDsXgfzgD9hpR167xvnUThYvuIr0vgrNr2LG3dUuP4evRN83ogzu8ipe9WY2n+YC0G3SU950Jd7oxOZTvoFuVvgPNNDbZMvz3XPyxZ0RXB2YSUcL01GY/s5ky5OEL9P8d2zVv0Z5OcQ6ol3QQUb5GWTNMeYfLLX6Pt8mOe4oEyf1vkZhx53W1hyO0JkeqeGh0Na9fmsHTltpQXe5+jf3gHlLR/aY8xX0GwIf9LB4vq3qHhKl50YKT4wbsP+oHqZ/Xrh797vDrU8TMvEDjPr5r7FdR7AT3Kl++AF57GdQIxzB/PmllsnSuYjzEs655rg/GSE+3i9fiNDblNWd3rBuc8vB/XXar3Dmvh/pYAtY6UCvfZvjp7y5sXW/UrPq/R06az0fuyKM79GWcMLnrejqWYIuVtioSRfzca7h/9RI55l3lJCb08XNicLmL+eoY8N+GvOOCT9NIP+aGfPLa+dbYh7EOWcTIU/KAXo7btA10NPL8mZveNMtzVsq5bbuzToPvccx69a6cYtqglq0eKS8QK/bSnwj1RqPbFgWZ33khupV1cxc7dtrtVjv3CnvlX7RFo09xWN/kT+Yai2q2XI33NZCuvLVWnn6YrtJuUDC9hchSTwdR5e25IojVl3Japju1qajHNrEtZj1r84z2qm+ppBrlCNjvtnNF0Yw8tt5gI44P7pf3CbZFVmcL3S/Rpf5Hf/IrXWrtapRvUmT6jVatbi/6TQXyFkF1bzphha3aJ949RpDW624Z30V+Eh81YhsKR+nmfRcaevhMWqYfiSvk4eX7DJx+QcazYj2r85rwFOhfxLK127u5K7BIQ7l4XaXfCE67lIFPfhZUlojBn0JXEQQyJNCkzjoV2mUvkGisDd23hnpKPWe19vjq4Z0mnjn1O8J7d9h4T++//6D5cvmd3H/3MlzDJyffFQ6Ynxuvt4Uty8KLiII5FFt1Pa4/Fwp8+5KoGH6zi8p6bvz7vPqlft0WjrxxTuXr107derUtWuXv7hs4vy3Qj0+q3ce4fxTnfOjs6S0ekJuPjPoGufd4BKCQF6oJ4/SaXtcfm60SjoL06lNPzLbpwBLk9cHT8CcF0r2fIA0PCYyV8jEbbd/xjMI9KdUimzSEenzpFKWa8xJPSFHvfdVfRx5L11W5XG/HXXE7SwoOHpULqzFbcoVe2Wi4QKCQF5J6HhnznumfMQwSvpORrpXgbrvWnMxT4rPNc6PH31Sjukzc8VM3AK4fCCQd4rNNjrv4+VD2i5UvHeN9Nl296N1eSJPsedoHOSTsdJB4xHnvLSWFA6XDwTyUu3EEhslPU0+JGIlt+mkc0YL1LcMtvNNhGUQc15Y8jmPzzV7PkDmfHSu5LjHwMUDgbxWum7SaZiem6vsWBj0KCOd9MihlFzebPtWjfWek0fTcCzfjv32lfIQ54RzUiYOHHcQyAdFZiv9cZpJ36rWp0dT0lE9nSbf895ZY88bGLYKmXOabqeco4R7B7kcF5MrZ9xD4dKBQD6oGwL9EDXplPRN6o4IqScp6SQlh933vDdt8N+HPXGRmnMSnnO/fZR8XMq5YhqgE8d9LFw4EKg0znsOct43qAYzYhYeLcVSciU4J5e3qnSoB/Ve9SvjHJtznfPR8pHJ53LFjPv2dnDZQCDf1FbPx6HMOw7Ti/sZ+lxGc5uuka6579h/z3uzt9+z+frM3HJRNOd6uv3o0TEq58VkLQt13PvCVQOBfBXuhD1A923Bzrtm07saDkseIKGOjbqG+paZ/jTQhI6Z+yvDXDTnmPOVsRb2nDruUFkDgfxQoiHznm9GelivAp103X/Py9u22DfyQgdPmEMxZ0U1uvwcc95LcRJSi4pRYQ2tZdlxFbXKRMIlA4F8V1hPkXSSkCsuNpLuihyAA3WUfdf9d8T6xbmLvbXrXRbP/g+hnGIumfOC48cVt90VU5TLOmWI494GrhgI5I+ihkgNcqSabmLTXa5BklHX/HfqwGusr8ro3cV9wB46efGEN38VKMeY42Q7N+ePqs5BmmbP9TVrGucD4XqBQP4pGoOOV7dc3cG9934my8PiRmHSuf+OSm3UrOddvPjy3JmLO/Ux1LjD4oYPXjNh25xfEeUq5sycoy6Zhao5d43mnNNEHAybAIH8VjvdpGObjhNyuc+axd6Rs6hR56hzDx7DfvHillVzZ0/ImHmfppkzMybMfeLld34ljGPINcqRz174Bcecue29DF8RXVXOe8LaVBDIfy2Qw3RK+iLTGS5jBhwTUf8co47tOqUd447Bxn/REceQY8oLNcpFzDW3fZZhJF34BoHzHdp30PakCLhUIFAp1NeQkEPe+9YU04N7D6D+O0admXXdibcQdtixy46NOQ/OkTkfkGo4S/I8kXOUcB8Cw6NAoFIpKN1IumbTz6WZH957pYy6xvoXhRz2IzLw9CcYcuSyf8Eo/4xjnmA8Rdovqj3fDp2vIFApFdwTNb2rNr24eLzF8cM6ENRRWo6wTmHHtBNRE76bIo5MOXPZPWEe3LVI5BwH6ClwlUCg0io8XonTSe69eEOcxROiei0UWNdh/6KkpLCQEU/+VlLCGf/0E4w5Ds1RbF7QIdXktSMXFaF+ONFv3w6zo0AgGxQXT4tscpxevDXV6hmhCR12EtSxD49YR7AT3CXhn2KHHUNOjfmxgoW9THeFGF1UdK6Y9skA5yCQ7TYdL1lVbPq5cdbPiR20UmcdW3aMOwWe8f0pZVxz2L/klB8r6NDbdF151OqionO5Oud7gHMQyEaF9MTDIiXvHafknnWX7h4+aNZOBvtnFHaGu444YZxAjig/3iHNolammXMangt5OOAcBLJNwem8RY73yOGUXJH7JeBxvR8bsHMnw/1LYt51kR+xAzTMBzzW2yryT95E3Ha2jgXy7SCQ/VqCE3KyTcfu+wZPNewuCb1mcZaxdefaKTB+bOVjCcMtm+Ljuv5SJKXh9iDOO7aFCwMC2aoFUjcsD9RzzxWN8zxkIjw5IePRlQt3mkhDfECHXmnD4tw8PWhQEXHbeXiOF6DHQz8cCGS3RmDvHZN+VTbqi7wMlMOjknunzez12KhRjyKNeqxXxqCE1MkRnvrUYxbp5lyw59NhEiQIZL9ShnDSJfc999y5TAd96OjVCHPmtuvp9kS4IiCQE4qIpz1yLFBn2ffic0VdY8sAc+q24/L5CLgeIJBDSqRxOlmgLhh1DfVE+1EPS1j9i8Z5MU22i2k4SLeDQM6pjd4kR913atQ1/72oq70OfPjoeRrluKaWSzEn3XAfbU+HOZAgkJOK7UkDdTH7zq16ZqptJ0oe9QvHHHvtujmHsVEgkOPqppN+VTfq+SQrV7RotB0efMSg1QRzFpyL5rwjrFYDgZxX2/hDevZdRx078JpVL+qXULqdFuMSMjXKf6HWnHjtgjlfAlU1EKisjTqK1CX/HZn1rV1j4vwNDdI0yokxF3NwKNmOOR8C+yKDQGUWqafzOhtNv1PUSayumfWt/QZF+jqwMS513GpCedG5XBabs5oaNeeQhQOBylAxHXWjLqKOHXjCetGirmmRXvrZQVEx4zKLCOTImOfy2JxijqPzJIjOQaCyVVi3Q2SVix6q01g9H6NOWC+alzk+LSXWjW0PikhOGNd10S/MlBOXPTdfwpyY825B8LGDQGWtiCXIqJOkHEZdN+uUdQ12THvRosxR49ISUiPbxkbEhWuKi4iNikyNSRuXmLlpK2OcQy5Qzpx2tFcqLGEBgcpFUdOp/65n5ahVp6zj3BxhvegXrK3z5s1bNG/eVvKvX4qYu65TziJz2Zr3jISPGwQqL0Wm6/67btWpWc8nhh278Zx3VfghjXHRlhPKBWseD4NkQKDyRV2y6gh1wjoz7MSya7DnnsPmXRL6CsgldpxDLlBOUnDb46GkBgKVu9ou0UnXUKdmnfvwAu1WymeMM8q5z46cdrDmIFBAKKJbR+LA6x48g12kPd8APP5ZjhFySjlKwcEyNRAoYBQck07NumDXCeyIdh13c+2ljGPIdco7DoyFjxYECihFDUwSUJdZx7DvtUKcQs4ov0oxnx4TDJ8qCBRwCkpJHLJPYJ3DznDnyIt4c8QR5MSWowRcOzDmIFCgKjQ6sSNlncB+FdHOcJeR3yX8FB30ETPl23t2gznOIFBgK2zswJ77JMOumfariHYJeMo3ZvwqM+Qa5UOmj4DdzkGgCqGImAXxzLBz2qkzL0p8CB2aPjA6BD49EKgiwR7drW8Sh13m/SOV8O3bk/q2S4E1qCBQRVRwVHS7BdOThjDeBRPP/zUkfvqCEdGxsDINBKrYCoqIjB7RbcGSvuk945M6YiXF90zvm7igW5vothGAOAhURvr/SORZlCmwWRcAAAAASUVORK5CYII=)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2-XLsLHoPXYv"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "\n",
        "!apt-get update && apt-get install -y xvfb\n",
        "!pip install swig\n",
        "!pip install gym[box2d]==0.23.1 pytorch-lightning==1.6.0 pyvirtualdisplay optuna==2.7.0"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZOSJl-X7zvs4"
      },
      "source": [
        "#### Setup virtual display"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FllWJVrvPhzz"
      },
      "outputs": [],
      "source": [
        "from pyvirtualdisplay import Display\n",
        "Display(visible=False, size=(1400, 900)).start()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cz8DLleGz_TF"
      },
      "source": [
        "#### Import the necessary code libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "id": "iVXUA1JoPmVM"
      },
      "outputs": [],
      "source": [
        "import copy\n",
        "import statistics\n",
        "import gym\n",
        "import torch\n",
        "import optuna\n",
        "\n",
        "import numpy as np\n",
        "import torch.nn.functional as F\n",
        "\n",
        "from collections import deque, namedtuple\n",
        "from IPython.display import HTML\n",
        "from base64 import b64encode\n",
        "\n",
        "from torch import nn\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data.dataset import IterableDataset\n",
        "from torch.optim import AdamW\n",
        "\n",
        "\n",
        "from lightning.pytorch import LightningModule, Trainer\n",
        "#earlier it was \n",
        "# from pytorch_lightning import LightningModule, Trainer\n",
        "#it raised the following error\n",
        "# https://github.com/optuna/optuna/issues/4689\n",
        "\n",
        "from gym.wrappers import RecordVideo, RecordEpisodeStatistics\n",
        "\n",
        "from optuna.integration.pytorch_lightning import PyTorchLightningPruningCallback\n",
        "# from pytorch_lightning.loggers import TensorBoardLogger\n",
        "\n",
        "device = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n",
        "num_gpus = torch.cuda.device_count()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "XPGaTA9mPmXn"
      },
      "outputs": [],
      "source": [
        "def display_video(episode=0):\n",
        "  video_file = open(f'/content/videos/rl-video-episode-{episode}.mp4', \"r+b\").read()\n",
        "  video_url = f\"data:video/mp4;base64,{b64encode(video_file).decode()}\"\n",
        "  return HTML(f\"<video width=600 controls><source src='{video_url}'></video>\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eLH52SgC0RRI"
      },
      "source": [
        "#### Create the Deep Q-Network"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "BL3UviKCPmZ6"
      },
      "outputs": [],
      "source": [
        "class DQN(nn.Module):\n",
        "    def __init__(self, hidden_size, obs_size, n_actions):\n",
        "        super().__init__()\n",
        "        self.net = nn.Sequential(\n",
        "            nn.Linear(obs_size, hidden_size),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(hidden_size, hidden_size),\n",
        "            nn.ReLU(),           \n",
        "            nn.Linear(hidden_size, n_actions),\n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.net(x.float())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bnk0wSWj0hAz"
      },
      "source": [
        "#### Create the policy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "n0DzHUT_PmcT"
      },
      "outputs": [],
      "source": [
        "def epsilon_greedy(state, env, net, epsilon=0.0):\n",
        "  if np.random.random() < epsilon:\n",
        "      action = env.action_space.sample()\n",
        "  else:\n",
        "      state = torch.tensor([state]).to(device)\n",
        "      q_values = net(state)\n",
        "      _, action = torch.max(q_values, dim=1)\n",
        "      action = int(action.item())\n",
        "  return action"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "brJmKGkl0jge"
      },
      "source": [
        "#### Create the replay buffer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "9T-fsyriPme6"
      },
      "outputs": [],
      "source": [
        "Experience = namedtuple(\n",
        "    \"Experience\",\n",
        "    field_names=[\"state\", \"action\", \"reward\", \"done\", \"new_state\"],\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {
        "id": "nLprQhrtPmhR"
      },
      "outputs": [],
      "source": [
        "class ReplayBuffer:\n",
        "\n",
        "    def __init__(self, capacity):\n",
        "        self.buffer = deque(maxlen=capacity)\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.buffer)\n",
        "\n",
        "    def append(self, experience):\n",
        "\n",
        "        self.buffer.append(experience)\n",
        "\n",
        "    def sample(self, batch_size):\n",
        "        indices = np.random.choice(len(self.buffer), batch_size, replace=False)\n",
        "        states, actions, rewards, dones, next_states = zip(*(self.buffer[idx] for idx in indices))\n",
        "\n",
        "        return (\n",
        "            np.array(states),\n",
        "            np.array(actions),\n",
        "            np.array(rewards, dtype=np.float32),\n",
        "            np.array(dones, dtype=bool),\n",
        "            np.array(next_states),\n",
        "        )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "id": "_BQjkQOlPmkU"
      },
      "outputs": [],
      "source": [
        "class RLDataset(IterableDataset):\n",
        "\n",
        "    def __init__(self, buffer, sample_size=200):\n",
        "        self.buffer = buffer\n",
        "        self.sample_size = sample_size\n",
        "\n",
        "    def __iter__(self):\n",
        "        states, actions, rewards, dones, new_states = self.buffer.sample(self.sample_size)\n",
        "        for i in range(len(dones)):\n",
        "            yield states[i], actions[i], rewards[i], dones[i], new_states[i]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0yvDC9qF0oPr"
      },
      "source": [
        "#### Create the environment"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "6KlHL-kznY3C"
      },
      "outputs": [],
      "source": [
        "def create_environment(name):\n",
        "  env = gym.make(name)\n",
        "  env = RecordVideo(env, video_folder='./videos', episode_trigger=lambda x: x % 50 == 0)\n",
        "  env = RecordEpisodeStatistics(env)\n",
        "  return env"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WLmsqFuW1kkC"
      },
      "source": [
        "#### Create the test/sampling function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "hwnhC_zCwKDm"
      },
      "outputs": [],
      "source": [
        "  @torch.no_grad()\n",
        "  def play_episode(env, q_net, buffer, policy=None, epsilon=0.):\n",
        "      obs = env.reset()\n",
        "      done = False\n",
        "\n",
        "      while not done:\n",
        "        if policy:\n",
        "          action = policy(obs, env, q_net, epsilon=epsilon)\n",
        "        else:\n",
        "          action = env.action_space.sample()\n",
        "          \n",
        "        next_obs, reward, done, info = env.step(action)\n",
        "        exp = Experience(obs, action, reward, done, next_obs)\n",
        "        buffer.append(exp)\n",
        "        obs = next_obs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sgXi6A4Z1p75"
      },
      "source": [
        "#### Create the Deep Q-Learning algorithm"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "metadata": {
        "id": "tOmxUJ1vnY5d"
      },
      "outputs": [],
      "source": [
        "class DeepQLearning(LightningModule):\n",
        "\n",
        "  def __init__(self, env_name, policy=epsilon_greedy, sample_fn=play_episode, \n",
        "               capacity=100_000, batch_size=256, lr=1e-3, hidden_size=128, gamma=0.99, \n",
        "               loss_fn=F.smooth_l1_loss, optim=AdamW, eps_start=1.0, eps_end=0.15, \n",
        "               eps_last_episode=100, samples_per_epoch=10_000, sync_rate=10):\n",
        "\n",
        "    super().__init__()\n",
        "    self.env = create_environment(env_name)\n",
        "\n",
        "    obs_size = self.env.observation_space.shape[0]\n",
        "    n_actions = self.env.action_space.n\n",
        "\n",
        "    self.q_net = DQN(hidden_size, obs_size, n_actions).to(device)\n",
        "    self.target_q_net = copy.deepcopy(self.q_net)\n",
        "    self.policy = policy\n",
        "\n",
        "    self.buffer = ReplayBuffer(capacity=capacity)\n",
        "\n",
        "    self.save_hyperparameters()\n",
        "\n",
        "    while len(self.buffer) < self.hparams.samples_per_epoch:\n",
        "\n",
        "      print(f\"{len(self.buffer)} samples in experience buffer. Filling...\")\n",
        "      self.hparams.sample_fn(self.env, self.q_net, self.buffer, epsilon=self.hparams.eps_start)\n",
        "  \n",
        "  def forward(self, x):\n",
        "    output = self.q_net(x)\n",
        "    return output\n",
        "\n",
        "  def configure_optimizers(self):\n",
        "    q_net_optimizer = self.hparams.optim(self.q_net.parameters(), lr=self.hparams.lr)\n",
        "    return [q_net_optimizer]\n",
        "\n",
        "  def train_dataloader(self):\n",
        "    dataset = RLDataset(self.buffer, self.hparams.samples_per_epoch)\n",
        "    dataloader = DataLoader(\n",
        "        dataset=dataset,\n",
        "        batch_size=self.hparams.batch_size,\n",
        "    )\n",
        "    return dataloader\n",
        "\n",
        "  def training_step(self, batch, batch_idx):\n",
        "\n",
        "    states, actions, rewards, dones, next_states = batch\n",
        "    actions=actions.long()#this line was added because of an error in the gather function below\n",
        "    actions = actions.unsqueeze(1)\n",
        "    rewards = rewards.unsqueeze(1)\n",
        "    dones = dones.unsqueeze(1)\n",
        "    state_action_values = self.q_net(states).gather(1, actions)\n",
        "\n",
        "    with torch.no_grad():\n",
        "      next_state_values, _ = self.target_q_net(next_states).max(dim=1, keepdim=True)\n",
        "      next_state_values[dones] = 0.0\n",
        "    \n",
        "    expected_state_action_values = rewards + self.hparams.gamma * next_state_values\n",
        "\n",
        "    loss = self.hparams.loss_fn(state_action_values, expected_state_action_values)\n",
        "    self.log('episode/MSE Loss', loss, on_step=False, on_epoch=True)\n",
        "    return loss\n",
        "\n",
        "\n",
        "  def on_train_epoch_end(self):\n",
        "\n",
        "    epsilon = max(\n",
        "        self.hparams.eps_end,\n",
        "        self.hparams.eps_start - self.current_epoch / self.hparams.eps_last_episode\n",
        "    )\n",
        "\n",
        "    self.hparams.sample_fn(self.env, self.q_net, self.buffer, policy=self.policy, epsilon=epsilon)\n",
        "\n",
        "    self.log(\"episode/Return\", self.env.return_queue[-1])\n",
        "    if self.current_epoch == 199:\n",
        "      returns = list(self.env.return_queue)\n",
        "      self.log(\"hp_metric\", statistics.mean(returns))#we are taking the mean of the return over 200 epsiodes for a certain set of parameters because return of each episode can vary greatly\n",
        "      #in comparison to each other\n",
        "      #this is the value that the obejctive function will return and study object will monitor\n",
        "    \n",
        "\n",
        "    if self.current_epoch % self.hparams.sync_rate == 0:\n",
        "      self.target_q_net.load_state_dict(self.q_net.state_dict())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iv-ZRdyf4mCt"
      },
      "source": [
        "#### Create the objective function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "metadata": {
        "id": "3B9ZoVV-QHrK"
      },
      "outputs": [],
      "source": [
        "def objective(trial):#trial is the object which we will use to select the parameters to test\n",
        "  lr = trial.suggest_float(\"lr\", 1e-5, 1e-1, log=True)#text in the \"\" is the name given to the parameter, the second and third parameter are the ranges\n",
        "  #between which a value will be selected, we also give it logarithmic scale because otherwise it will spend a lot of time searching for smaller values within this range \n",
        "  gamma = trial.suggest_float(\"gamma\", 0.0, 1.0)\n",
        "  hidden_size = trial.suggest_categorical(\"hidden_size\", [32, 64, 128, 256])\n",
        "  eps_end = trial.suggest_float(\"eps_end\", 0.0, 0.3)\n",
        "\n",
        "  algo = DeepQLearning(\n",
        "      'LunarLander-v2',\n",
        "      lr=lr,\n",
        "      gamma=gamma,\n",
        "      hidden_size=hidden_size,\n",
        "      eps_end=eps_end\n",
        "  )\n",
        "\n",
        "  callback = PyTorchLightningPruningCallback(trial, monitor=\"episode/Return\")\n",
        "\n",
        "  # logger = TensorBoardLogger(\"tb_logs\", name=\"my_model\")\n",
        "\n",
        "  trainer = Trainer(\n",
        "    #   gpus=num_gpus, \n",
        "      max_epochs=200,\n",
        "      # track_grad_norm=2,\n",
        "      callbacks=[callback],\n",
        "      # logger=logger\n",
        "  )\n",
        "\n",
        "  hyperparameters = dict(\n",
        "      lr=lr, \n",
        "      gamma=gamma, \n",
        "      hidden_size=hidden_size,\n",
        "      eps_end=eps_end\n",
        "  )\n",
        "  \n",
        "  trainer.logger.log_hyperparams(hyperparameters)\n",
        "  trainer.fit(algo)\n",
        "\n",
        "  return trainer.callback_metrics[\"episode/Return\"].item()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mJUxWPhF47mW"
      },
      "source": [
        "#### Create the optimization study"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 46,
      "metadata": {
        "id": "1MynMxzZQHtz"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[I 2024-07-02 08:06:46,543] A new study created in memory with name: no-name-1a5545f0-db47-433d-b795-d943950ff53c\n"
          ]
        }
      ],
      "source": [
        "pruner = optuna.pruners.SuccessiveHalvingPruner()\n",
        "study = optuna.create_study(direction=\"maximize\", pruner=pruner)#direction=\"maximise\" is telling the function that we wish to maximize the value which we are monitoring inside the objective function\n",
        "#which is the running average of the returns"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6mm9P0sX1wAA"
      },
      "source": [
        "#### Purge logs and run the visualization tool (Tensorboard)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MfGQdpn0nY99"
      },
      "outputs": [],
      "source": [
        "# Start tensorboard.\n",
        "!rm -r /content/lightning_logs/\n",
        "!rm -r /content/videos/\n",
        "%load_ext tensorboard\n",
        "%tensorboard --logdir /content/lightning_logs/"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9v1eHYSJ5EpO"
      },
      "source": [
        "#### Run the hyperparameter search"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 47,
      "metadata": {
        "id": "Ig8c_RM8nZLN"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "96 samples in experience buffer. Filling...\n",
            "156 samples in experience buffer. Filling...\n",
            "243 samples in experience buffer. Filling...\n",
            "374 samples in experience buffer. Filling...\n",
            "480 samples in experience buffer. Filling...\n",
            "578 samples in experience buffer. Filling...\n",
            "691 samples in experience buffer. Filling...\n",
            "819 samples in experience buffer. Filling...\n",
            "909 samples in experience buffer. Filling...\n",
            "1000 samples in experience buffer. Filling...\n",
            "1099 samples in experience buffer. Filling...\n",
            "1185 samples in experience buffer. Filling...\n",
            "1312 samples in experience buffer. Filling...\n",
            "1415 samples in experience buffer. Filling...\n",
            "1541 samples in experience buffer. Filling...\n",
            "1622 samples in experience buffer. Filling...\n",
            "1734 samples in experience buffer. Filling...\n",
            "1797 samples in experience buffer. Filling...\n",
            "1904 samples in experience buffer. Filling...\n",
            "1975 samples in experience buffer. Filling...\n",
            "2099 samples in experience buffer. Filling...\n",
            "2195 samples in experience buffer. Filling...\n",
            "2308 samples in experience buffer. Filling...\n",
            "2381 samples in experience buffer. Filling...\n",
            "2493 samples in experience buffer. Filling...\n",
            "2598 samples in experience buffer. Filling...\n",
            "2678 samples in experience buffer. Filling...\n",
            "2816 samples in experience buffer. Filling...\n",
            "2929 samples in experience buffer. Filling...\n",
            "3042 samples in experience buffer. Filling...\n",
            "3113 samples in experience buffer. Filling...\n",
            "3179 samples in experience buffer. Filling...\n",
            "3254 samples in experience buffer. Filling...\n",
            "3330 samples in experience buffer. Filling...\n",
            "3400 samples in experience buffer. Filling...\n",
            "3476 samples in experience buffer. Filling...\n",
            "3593 samples in experience buffer. Filling...\n",
            "3658 samples in experience buffer. Filling...\n",
            "3777 samples in experience buffer. Filling...\n",
            "3873 samples in experience buffer. Filling...\n",
            "3985 samples in experience buffer. Filling...\n",
            "4077 samples in experience buffer. Filling...\n",
            "4165 samples in experience buffer. Filling...\n",
            "4268 samples in experience buffer. Filling...\n",
            "4325 samples in experience buffer. Filling...\n",
            "4395 samples in experience buffer. Filling...\n",
            "4475 samples in experience buffer. Filling...\n",
            "4562 samples in experience buffer. Filling...\n",
            "4650 samples in experience buffer. Filling...\n",
            "4732 samples in experience buffer. Filling...\n",
            "4810 samples in experience buffer. Filling...\n",
            "4880 samples in experience buffer. Filling...\n",
            "4961 samples in experience buffer. Filling...\n",
            "5046 samples in experience buffer. Filling...\n",
            "5133 samples in experience buffer. Filling...\n",
            "5232 samples in experience buffer. Filling...\n",
            "5302 samples in experience buffer. Filling...\n",
            "5395 samples in experience buffer. Filling...\n",
            "5465 samples in experience buffer. Filling...\n",
            "5596 samples in experience buffer. Filling...\n",
            "5707 samples in experience buffer. Filling...\n",
            "5814 samples in experience buffer. Filling...\n",
            "5876 samples in experience buffer. Filling...\n",
            "5996 samples in experience buffer. Filling...\n",
            "6115 samples in experience buffer. Filling...\n",
            "6199 samples in experience buffer. Filling...\n",
            "6295 samples in experience buffer. Filling...\n",
            "6395 samples in experience buffer. Filling...\n",
            "6468 samples in experience buffer. Filling...\n",
            "6536 samples in experience buffer. Filling...\n",
            "6621 samples in experience buffer. Filling...\n",
            "6748 samples in experience buffer. Filling...\n",
            "6847 samples in experience buffer. Filling...\n",
            "6959 samples in experience buffer. Filling...\n",
            "7021 samples in experience buffer. Filling...\n",
            "7092 samples in experience buffer. Filling...\n",
            "7178 samples in experience buffer. Filling...\n",
            "7264 samples in experience buffer. Filling...\n",
            "7328 samples in experience buffer. Filling...\n",
            "7398 samples in experience buffer. Filling...\n",
            "7509 samples in experience buffer. Filling...\n",
            "7626 samples in experience buffer. Filling...\n",
            "7705 samples in experience buffer. Filling...\n",
            "7828 samples in experience buffer. Filling...\n",
            "7903 samples in experience buffer. Filling...\n",
            "7986 samples in experience buffer. Filling...\n",
            "8051 samples in experience buffer. Filling...\n",
            "8131 samples in experience buffer. Filling...\n",
            "8232 samples in experience buffer. Filling...\n",
            "8337 samples in experience buffer. Filling...\n",
            "8419 samples in experience buffer. Filling...\n",
            "8512 samples in experience buffer. Filling...\n",
            "8641 samples in experience buffer. Filling...\n",
            "8733 samples in experience buffer. Filling...\n",
            "8799 samples in experience buffer. Filling...\n",
            "8862 samples in experience buffer. Filling...\n",
            "8969 samples in experience buffer. Filling...\n",
            "9047 samples in experience buffer. Filling...\n",
            "9173 samples in experience buffer. Filling...\n",
            "9305 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "9423 samples in experience buffer. Filling...\n",
            "9508 samples in experience buffer. Filling...\n",
            "9636 samples in experience buffer. Filling...\n",
            "9716 samples in experience buffer. Filling...\n",
            "9789 samples in experience buffer. Filling...\n",
            "9902 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 69.1 K | train\n",
            "1 | target_q_net | DQN  | 69.1 K | train\n",
            "----------------------------------------------\n",
            "138 K     Trainable params\n",
            "0         Non-trainable params\n",
            "138 K     Total params\n",
            "0.553     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c755bd0be27446768d4802dfef6158b2",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\haadi\\AppData\\Local\\Temp\\ipykernel_24364\\1507904788.py:5: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ..\\torch\\csrc\\utils\\tensor_new.cpp:248.)\n",
            "  state = torch.tensor([state]).to(device)\n",
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 08:10:47,971] Trial 0 finished with value: -241.96539306640625 and parameters: {'lr': 0.0017503780778528849, 'gamma': 0.7781599075447769, 'hidden_size': 256, 'eps_end': 0.10772209039861792}. Best is trial 0 with value: -241.96539306640625.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "83 samples in experience buffer. Filling...\n",
            "149 samples in experience buffer. Filling...\n",
            "238 samples in experience buffer. Filling...\n",
            "317 samples in experience buffer. Filling...\n",
            "420 samples in experience buffer. Filling...\n",
            "552 samples in experience buffer. Filling...\n",
            "675 samples in experience buffer. Filling...\n",
            "747 samples in experience buffer. Filling...\n",
            "831 samples in experience buffer. Filling...\n",
            "898 samples in experience buffer. Filling...\n",
            "991 samples in experience buffer. Filling...\n",
            "1088 samples in experience buffer. Filling...\n",
            "1165 samples in experience buffer. Filling...\n",
            "1287 samples in experience buffer. Filling...\n",
            "1372 samples in experience buffer. Filling...\n",
            "1437 samples in experience buffer. Filling...\n",
            "1520 samples in experience buffer. Filling...\n",
            "1589 samples in experience buffer. Filling...\n",
            "1699 samples in experience buffer. Filling...\n",
            "1805 samples in experience buffer. Filling...\n",
            "1903 samples in experience buffer. Filling...\n",
            "1985 samples in experience buffer. Filling...\n",
            "2067 samples in experience buffer. Filling...\n",
            "2155 samples in experience buffer. Filling...\n",
            "2273 samples in experience buffer. Filling...\n",
            "2370 samples in experience buffer. Filling...\n",
            "2459 samples in experience buffer. Filling...\n",
            "2516 samples in experience buffer. Filling...\n",
            "2630 samples in experience buffer. Filling...\n",
            "2688 samples in experience buffer. Filling...\n",
            "2785 samples in experience buffer. Filling...\n",
            "2866 samples in experience buffer. Filling...\n",
            "2961 samples in experience buffer. Filling...\n",
            "3047 samples in experience buffer. Filling...\n",
            "3138 samples in experience buffer. Filling...\n",
            "3256 samples in experience buffer. Filling...\n",
            "3369 samples in experience buffer. Filling...\n",
            "3448 samples in experience buffer. Filling...\n",
            "3555 samples in experience buffer. Filling...\n",
            "3654 samples in experience buffer. Filling...\n",
            "3729 samples in experience buffer. Filling...\n",
            "3805 samples in experience buffer. Filling...\n",
            "3913 samples in experience buffer. Filling...\n",
            "4050 samples in experience buffer. Filling...\n",
            "4161 samples in experience buffer. Filling...\n",
            "4282 samples in experience buffer. Filling...\n",
            "4364 samples in experience buffer. Filling...\n",
            "4435 samples in experience buffer. Filling...\n",
            "4545 samples in experience buffer. Filling...\n",
            "4612 samples in experience buffer. Filling...\n",
            "4703 samples in experience buffer. Filling...\n",
            "4793 samples in experience buffer. Filling...\n",
            "4881 samples in experience buffer. Filling...\n",
            "4977 samples in experience buffer. Filling...\n",
            "5089 samples in experience buffer. Filling...\n",
            "5160 samples in experience buffer. Filling...\n",
            "5230 samples in experience buffer. Filling...\n",
            "5300 samples in experience buffer. Filling...\n",
            "5368 samples in experience buffer. Filling...\n",
            "5478 samples in experience buffer. Filling...\n",
            "5612 samples in experience buffer. Filling...\n",
            "5707 samples in experience buffer. Filling...\n",
            "5781 samples in experience buffer. Filling...\n",
            "5871 samples in experience buffer. Filling...\n",
            "6003 samples in experience buffer. Filling...\n",
            "6080 samples in experience buffer. Filling...\n",
            "6182 samples in experience buffer. Filling...\n",
            "6257 samples in experience buffer. Filling...\n",
            "6356 samples in experience buffer. Filling...\n",
            "6422 samples in experience buffer. Filling...\n",
            "6515 samples in experience buffer. Filling...\n",
            "6622 samples in experience buffer. Filling...\n",
            "6744 samples in experience buffer. Filling...\n",
            "6805 samples in experience buffer. Filling...\n",
            "6872 samples in experience buffer. Filling...\n",
            "6969 samples in experience buffer. Filling...\n",
            "7064 samples in experience buffer. Filling...\n",
            "7152 samples in experience buffer. Filling...\n",
            "7221 samples in experience buffer. Filling...\n",
            "7286 samples in experience buffer. Filling...\n",
            "7369 samples in experience buffer. Filling...\n",
            "7493 samples in experience buffer. Filling...\n",
            "7598 samples in experience buffer. Filling...\n",
            "7704 samples in experience buffer. Filling...\n",
            "7793 samples in experience buffer. Filling...\n",
            "7879 samples in experience buffer. Filling...\n",
            "7964 samples in experience buffer. Filling...\n",
            "8054 samples in experience buffer. Filling...\n",
            "8124 samples in experience buffer. Filling...\n",
            "8224 samples in experience buffer. Filling...\n",
            "8312 samples in experience buffer. Filling...\n",
            "8382 samples in experience buffer. Filling...\n",
            "8487 samples in experience buffer. Filling...\n",
            "8609 samples in experience buffer. Filling...\n",
            "8716 samples in experience buffer. Filling...\n",
            "8776 samples in experience buffer. Filling...\n",
            "8875 samples in experience buffer. Filling...\n",
            "8976 samples in experience buffer. Filling...\n",
            "9101 samples in experience buffer. Filling...\n",
            "9170 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "9277 samples in experience buffer. Filling...\n",
            "9376 samples in experience buffer. Filling...\n",
            "9487 samples in experience buffer. Filling...\n",
            "9564 samples in experience buffer. Filling...\n",
            "9682 samples in experience buffer. Filling...\n",
            "9754 samples in experience buffer. Filling...\n",
            "9854 samples in experience buffer. Filling...\n",
            "9913 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 18.2 K | train\n",
            "1 | target_q_net | DQN  | 18.2 K | train\n",
            "----------------------------------------------\n",
            "36.4 K    Trainable params\n",
            "0         Non-trainable params\n",
            "36.4 K    Total params\n",
            "0.145     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "13910aaaba4049dc9bbf903ad2923908",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 08:13:51,831] Trial 1 finished with value: -260.06988525390625 and parameters: {'lr': 0.00011680536902854591, 'gamma': 0.32221493974685844, 'hidden_size': 128, 'eps_end': 0.2272240674608202}. Best is trial 0 with value: -241.96539306640625.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "81 samples in experience buffer. Filling...\n",
            "151 samples in experience buffer. Filling...\n",
            "228 samples in experience buffer. Filling...\n",
            "341 samples in experience buffer. Filling...\n",
            "409 samples in experience buffer. Filling...\n",
            "499 samples in experience buffer. Filling...\n",
            "597 samples in experience buffer. Filling...\n",
            "696 samples in experience buffer. Filling...\n",
            "775 samples in experience buffer. Filling...\n",
            "866 samples in experience buffer. Filling...\n",
            "960 samples in experience buffer. Filling...\n",
            "1078 samples in experience buffer. Filling...\n",
            "1162 samples in experience buffer. Filling...\n",
            "1273 samples in experience buffer. Filling...\n",
            "1361 samples in experience buffer. Filling...\n",
            "1459 samples in experience buffer. Filling...\n",
            "1579 samples in experience buffer. Filling...\n",
            "1654 samples in experience buffer. Filling...\n",
            "1768 samples in experience buffer. Filling...\n",
            "1874 samples in experience buffer. Filling...\n",
            "1966 samples in experience buffer. Filling...\n",
            "2037 samples in experience buffer. Filling...\n",
            "2135 samples in experience buffer. Filling...\n",
            "2231 samples in experience buffer. Filling...\n",
            "2396 samples in experience buffer. Filling...\n",
            "2531 samples in experience buffer. Filling...\n",
            "2639 samples in experience buffer. Filling...\n",
            "2703 samples in experience buffer. Filling...\n",
            "2785 samples in experience buffer. Filling...\n",
            "2857 samples in experience buffer. Filling...\n",
            "2945 samples in experience buffer. Filling...\n",
            "3029 samples in experience buffer. Filling...\n",
            "3115 samples in experience buffer. Filling...\n",
            "3198 samples in experience buffer. Filling...\n",
            "3323 samples in experience buffer. Filling...\n",
            "3404 samples in experience buffer. Filling...\n",
            "3503 samples in experience buffer. Filling...\n",
            "3604 samples in experience buffer. Filling...\n",
            "3709 samples in experience buffer. Filling...\n",
            "3815 samples in experience buffer. Filling...\n",
            "3932 samples in experience buffer. Filling...\n",
            "4024 samples in experience buffer. Filling...\n",
            "4111 samples in experience buffer. Filling...\n",
            "4215 samples in experience buffer. Filling...\n",
            "4314 samples in experience buffer. Filling...\n",
            "4428 samples in experience buffer. Filling...\n",
            "4505 samples in experience buffer. Filling...\n",
            "4585 samples in experience buffer. Filling...\n",
            "4653 samples in experience buffer. Filling...\n",
            "4722 samples in experience buffer. Filling...\n",
            "4835 samples in experience buffer. Filling...\n",
            "4923 samples in experience buffer. Filling...\n",
            "5031 samples in experience buffer. Filling...\n",
            "5095 samples in experience buffer. Filling...\n",
            "5171 samples in experience buffer. Filling...\n",
            "5292 samples in experience buffer. Filling...\n",
            "5348 samples in experience buffer. Filling...\n",
            "5411 samples in experience buffer. Filling...\n",
            "5484 samples in experience buffer. Filling...\n",
            "5561 samples in experience buffer. Filling...\n",
            "5628 samples in experience buffer. Filling...\n",
            "5704 samples in experience buffer. Filling...\n",
            "5777 samples in experience buffer. Filling...\n",
            "5864 samples in experience buffer. Filling...\n",
            "5938 samples in experience buffer. Filling...\n",
            "6021 samples in experience buffer. Filling...\n",
            "6108 samples in experience buffer. Filling...\n",
            "6184 samples in experience buffer. Filling...\n",
            "6287 samples in experience buffer. Filling...\n",
            "6381 samples in experience buffer. Filling...\n",
            "6464 samples in experience buffer. Filling...\n",
            "6528 samples in experience buffer. Filling...\n",
            "6587 samples in experience buffer. Filling...\n",
            "6669 samples in experience buffer. Filling...\n",
            "6750 samples in experience buffer. Filling...\n",
            "6825 samples in experience buffer. Filling...\n",
            "6900 samples in experience buffer. Filling...\n",
            "6994 samples in experience buffer. Filling...\n",
            "7090 samples in experience buffer. Filling...\n",
            "7197 samples in experience buffer. Filling...\n",
            "7283 samples in experience buffer. Filling...\n",
            "7366 samples in experience buffer. Filling...\n",
            "7462 samples in experience buffer. Filling...\n",
            "7555 samples in experience buffer. Filling...\n",
            "7670 samples in experience buffer. Filling...\n",
            "7746 samples in experience buffer. Filling...\n",
            "7832 samples in experience buffer. Filling...\n",
            "7893 samples in experience buffer. Filling...\n",
            "7966 samples in experience buffer. Filling...\n",
            "8031 samples in experience buffer. Filling...\n",
            "8120 samples in experience buffer. Filling...\n",
            "8201 samples in experience buffer. Filling...\n",
            "8271 samples in experience buffer. Filling...\n",
            "8355 samples in experience buffer. Filling...\n",
            "8440 samples in experience buffer. Filling...\n",
            "8529 samples in experience buffer. Filling...\n",
            "8636 samples in experience buffer. Filling...\n",
            "8718 samples in experience buffer. Filling...\n",
            "8777 samples in experience buffer. Filling...\n",
            "8890 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "8982 samples in experience buffer. Filling...\n",
            "9072 samples in experience buffer. Filling...\n",
            "9158 samples in experience buffer. Filling...\n",
            "9288 samples in experience buffer. Filling...\n",
            "9395 samples in experience buffer. Filling...\n",
            "9498 samples in experience buffer. Filling...\n",
            "9571 samples in experience buffer. Filling...\n",
            "9659 samples in experience buffer. Filling...\n",
            "9732 samples in experience buffer. Filling...\n",
            "9809 samples in experience buffer. Filling...\n",
            "9908 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n",
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 69.1 K | train\n",
            "1 | target_q_net | DQN  | 69.1 K | train\n",
            "----------------------------------------------\n",
            "138 K     Trainable params\n",
            "0         Non-trainable params\n",
            "138 K     Total params\n",
            "0.553     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5f4921b3210e40b6a84b088559e46d4f",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 08:19:28,159] Trial 2 finished with value: -69.89277648925781 and parameters: {'lr': 0.001105918756644981, 'gamma': 0.9027112477363138, 'hidden_size': 256, 'eps_end': 0.06243170170120603}. Best is trial 2 with value: -69.89277648925781.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "77 samples in experience buffer. Filling...\n",
            "152 samples in experience buffer. Filling...\n",
            "251 samples in experience buffer. Filling...\n",
            "365 samples in experience buffer. Filling...\n",
            "474 samples in experience buffer. Filling...\n",
            "567 samples in experience buffer. Filling...\n",
            "686 samples in experience buffer. Filling...\n",
            "786 samples in experience buffer. Filling...\n",
            "909 samples in experience buffer. Filling...\n",
            "1035 samples in experience buffer. Filling...\n",
            "1125 samples in experience buffer. Filling...\n",
            "1216 samples in experience buffer. Filling...\n",
            "1296 samples in experience buffer. Filling...\n",
            "1353 samples in experience buffer. Filling...\n",
            "1427 samples in experience buffer. Filling...\n",
            "1508 samples in experience buffer. Filling...\n",
            "1575 samples in experience buffer. Filling...\n",
            "1655 samples in experience buffer. Filling...\n",
            "1730 samples in experience buffer. Filling...\n",
            "1846 samples in experience buffer. Filling...\n",
            "1929 samples in experience buffer. Filling...\n",
            "2035 samples in experience buffer. Filling...\n",
            "2116 samples in experience buffer. Filling...\n",
            "2201 samples in experience buffer. Filling...\n",
            "2288 samples in experience buffer. Filling...\n",
            "2354 samples in experience buffer. Filling...\n",
            "2416 samples in experience buffer. Filling...\n",
            "2539 samples in experience buffer. Filling...\n",
            "2622 samples in experience buffer. Filling...\n",
            "2697 samples in experience buffer. Filling...\n",
            "2817 samples in experience buffer. Filling...\n",
            "2930 samples in experience buffer. Filling...\n",
            "3004 samples in experience buffer. Filling...\n",
            "3090 samples in experience buffer. Filling...\n",
            "3213 samples in experience buffer. Filling...\n",
            "3280 samples in experience buffer. Filling...\n",
            "3351 samples in experience buffer. Filling...\n",
            "3422 samples in experience buffer. Filling...\n",
            "3493 samples in experience buffer. Filling...\n",
            "3600 samples in experience buffer. Filling...\n",
            "3695 samples in experience buffer. Filling...\n",
            "3801 samples in experience buffer. Filling...\n",
            "3899 samples in experience buffer. Filling...\n",
            "3979 samples in experience buffer. Filling...\n",
            "4075 samples in experience buffer. Filling...\n",
            "4200 samples in experience buffer. Filling...\n",
            "4316 samples in experience buffer. Filling...\n",
            "4382 samples in experience buffer. Filling...\n",
            "4487 samples in experience buffer. Filling...\n",
            "4548 samples in experience buffer. Filling...\n",
            "4635 samples in experience buffer. Filling...\n",
            "4723 samples in experience buffer. Filling...\n",
            "4784 samples in experience buffer. Filling...\n",
            "4877 samples in experience buffer. Filling...\n",
            "4946 samples in experience buffer. Filling...\n",
            "5060 samples in experience buffer. Filling...\n",
            "5132 samples in experience buffer. Filling...\n",
            "5217 samples in experience buffer. Filling...\n",
            "5276 samples in experience buffer. Filling...\n",
            "5351 samples in experience buffer. Filling...\n",
            "5434 samples in experience buffer. Filling...\n",
            "5514 samples in experience buffer. Filling...\n",
            "5603 samples in experience buffer. Filling...\n",
            "5676 samples in experience buffer. Filling...\n",
            "5735 samples in experience buffer. Filling...\n",
            "5803 samples in experience buffer. Filling...\n",
            "5914 samples in experience buffer. Filling...\n",
            "6004 samples in experience buffer. Filling...\n",
            "6087 samples in experience buffer. Filling...\n",
            "6161 samples in experience buffer. Filling...\n",
            "6235 samples in experience buffer. Filling...\n",
            "6314 samples in experience buffer. Filling...\n",
            "6405 samples in experience buffer. Filling...\n",
            "6524 samples in experience buffer. Filling...\n",
            "6632 samples in experience buffer. Filling...\n",
            "6733 samples in experience buffer. Filling...\n",
            "6849 samples in experience buffer. Filling...\n",
            "6944 samples in experience buffer. Filling...\n",
            "7039 samples in experience buffer. Filling...\n",
            "7140 samples in experience buffer. Filling...\n",
            "7234 samples in experience buffer. Filling...\n",
            "7359 samples in experience buffer. Filling...\n",
            "7471 samples in experience buffer. Filling...\n",
            "7553 samples in experience buffer. Filling...\n",
            "7664 samples in experience buffer. Filling...\n",
            "7753 samples in experience buffer. Filling...\n",
            "7815 samples in experience buffer. Filling...\n",
            "7924 samples in experience buffer. Filling...\n",
            "7991 samples in experience buffer. Filling...\n",
            "8097 samples in experience buffer. Filling...\n",
            "8223 samples in experience buffer. Filling...\n",
            "8334 samples in experience buffer. Filling...\n",
            "8419 samples in experience buffer. Filling...\n",
            "8522 samples in experience buffer. Filling...\n",
            "8623 samples in experience buffer. Filling...\n",
            "8701 samples in experience buffer. Filling...\n",
            "8807 samples in experience buffer. Filling...\n",
            "8872 samples in experience buffer. Filling...\n",
            "8939 samples in experience buffer. Filling...\n",
            "8998 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "9106 samples in experience buffer. Filling...\n",
            "9189 samples in experience buffer. Filling...\n",
            "9247 samples in experience buffer. Filling...\n",
            "9326 samples in experience buffer. Filling...\n",
            "9424 samples in experience buffer. Filling...\n",
            "9507 samples in experience buffer. Filling...\n",
            "9610 samples in experience buffer. Filling...\n",
            "9704 samples in experience buffer. Filling...\n",
            "9811 samples in experience buffer. Filling...\n",
            "9930 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n",
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 18.2 K | train\n",
            "1 | target_q_net | DQN  | 18.2 K | train\n",
            "----------------------------------------------\n",
            "36.4 K    Trainable params\n",
            "0         Non-trainable params\n",
            "36.4 K    Total params\n",
            "0.145     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b44ba88ca7d24e64adb31a8c2ff17cfa",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 08:22:41,821] Trial 3 finished with value: -93.09330749511719 and parameters: {'lr': 0.0026668941298903966, 'gamma': 0.359545869105662, 'hidden_size': 128, 'eps_end': 0.22715183732296815}. Best is trial 2 with value: -69.89277648925781.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "76 samples in experience buffer. Filling...\n",
            "162 samples in experience buffer. Filling...\n",
            "255 samples in experience buffer. Filling...\n",
            "337 samples in experience buffer. Filling...\n",
            "469 samples in experience buffer. Filling...\n",
            "583 samples in experience buffer. Filling...\n",
            "703 samples in experience buffer. Filling...\n",
            "800 samples in experience buffer. Filling...\n",
            "902 samples in experience buffer. Filling...\n",
            "996 samples in experience buffer. Filling...\n",
            "1092 samples in experience buffer. Filling...\n",
            "1211 samples in experience buffer. Filling...\n",
            "1291 samples in experience buffer. Filling...\n",
            "1407 samples in experience buffer. Filling...\n",
            "1495 samples in experience buffer. Filling...\n",
            "1558 samples in experience buffer. Filling...\n",
            "1668 samples in experience buffer. Filling...\n",
            "1753 samples in experience buffer. Filling...\n",
            "1843 samples in experience buffer. Filling...\n",
            "1965 samples in experience buffer. Filling...\n",
            "2053 samples in experience buffer. Filling...\n",
            "2127 samples in experience buffer. Filling...\n",
            "2202 samples in experience buffer. Filling...\n",
            "2270 samples in experience buffer. Filling...\n",
            "2343 samples in experience buffer. Filling...\n",
            "2482 samples in experience buffer. Filling...\n",
            "2543 samples in experience buffer. Filling...\n",
            "2662 samples in experience buffer. Filling...\n",
            "2786 samples in experience buffer. Filling...\n",
            "2906 samples in experience buffer. Filling...\n",
            "2995 samples in experience buffer. Filling...\n",
            "3073 samples in experience buffer. Filling...\n",
            "3145 samples in experience buffer. Filling...\n",
            "3225 samples in experience buffer. Filling...\n",
            "3333 samples in experience buffer. Filling...\n",
            "3404 samples in experience buffer. Filling...\n",
            "3490 samples in experience buffer. Filling...\n",
            "3555 samples in experience buffer. Filling...\n",
            "3647 samples in experience buffer. Filling...\n",
            "3763 samples in experience buffer. Filling...\n",
            "3834 samples in experience buffer. Filling...\n",
            "3903 samples in experience buffer. Filling...\n",
            "3996 samples in experience buffer. Filling...\n",
            "4083 samples in experience buffer. Filling...\n",
            "4166 samples in experience buffer. Filling...\n",
            "4288 samples in experience buffer. Filling...\n",
            "4378 samples in experience buffer. Filling...\n",
            "4464 samples in experience buffer. Filling...\n",
            "4600 samples in experience buffer. Filling...\n",
            "4684 samples in experience buffer. Filling...\n",
            "4766 samples in experience buffer. Filling...\n",
            "4858 samples in experience buffer. Filling...\n",
            "4983 samples in experience buffer. Filling...\n",
            "5041 samples in experience buffer. Filling...\n",
            "5120 samples in experience buffer. Filling...\n",
            "5206 samples in experience buffer. Filling...\n",
            "5270 samples in experience buffer. Filling...\n",
            "5361 samples in experience buffer. Filling...\n",
            "5452 samples in experience buffer. Filling...\n",
            "5540 samples in experience buffer. Filling...\n",
            "5619 samples in experience buffer. Filling...\n",
            "5718 samples in experience buffer. Filling...\n",
            "5820 samples in experience buffer. Filling...\n",
            "5895 samples in experience buffer. Filling...\n",
            "6009 samples in experience buffer. Filling...\n",
            "6082 samples in experience buffer. Filling...\n",
            "6158 samples in experience buffer. Filling...\n",
            "6246 samples in experience buffer. Filling...\n",
            "6321 samples in experience buffer. Filling...\n",
            "6445 samples in experience buffer. Filling...\n",
            "6526 samples in experience buffer. Filling...\n",
            "6595 samples in experience buffer. Filling...\n",
            "6688 samples in experience buffer. Filling...\n",
            "6770 samples in experience buffer. Filling...\n",
            "6857 samples in experience buffer. Filling...\n",
            "6941 samples in experience buffer. Filling...\n",
            "7022 samples in experience buffer. Filling...\n",
            "7124 samples in experience buffer. Filling...\n",
            "7207 samples in experience buffer. Filling...\n",
            "7306 samples in experience buffer. Filling...\n",
            "7448 samples in experience buffer. Filling...\n",
            "7560 samples in experience buffer. Filling...\n",
            "7669 samples in experience buffer. Filling...\n",
            "7738 samples in experience buffer. Filling...\n",
            "7834 samples in experience buffer. Filling...\n",
            "7913 samples in experience buffer. Filling...\n",
            "7973 samples in experience buffer. Filling...\n",
            "8072 samples in experience buffer. Filling...\n",
            "8158 samples in experience buffer. Filling...\n",
            "8268 samples in experience buffer. Filling...\n",
            "8334 samples in experience buffer. Filling...\n",
            "8457 samples in experience buffer. Filling...\n",
            "8573 samples in experience buffer. Filling...\n",
            "8672 samples in experience buffer. Filling...\n",
            "8768 samples in experience buffer. Filling...\n",
            "8864 samples in experience buffer. Filling...\n",
            "8945 samples in experience buffer. Filling...\n",
            "9081 samples in experience buffer. Filling...\n",
            "9193 samples in experience buffer. Filling...\n",
            "9273 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "9353 samples in experience buffer. Filling...\n",
            "9442 samples in experience buffer. Filling...\n",
            "9562 samples in experience buffer. Filling...\n",
            "9674 samples in experience buffer. Filling...\n",
            "9830 samples in experience buffer. Filling...\n",
            "9906 samples in experience buffer. Filling...\n",
            "9996 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 69.1 K | train\n",
            "1 | target_q_net | DQN  | 69.1 K | train\n",
            "----------------------------------------------\n",
            "138 K     Trainable params\n",
            "0         Non-trainable params\n",
            "138 K     Total params\n",
            "0.553     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0fc4759b3ae343f1adbe28e3ed4052a4",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 08:26:04,768] Trial 4 finished with value: -179.56101989746094 and parameters: {'lr': 0.0023922485145384358, 'gamma': 0.6302222399546862, 'hidden_size': 256, 'eps_end': 0.06673136205305802}. Best is trial 2 with value: -69.89277648925781.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "98 samples in experience buffer. Filling...\n",
            "156 samples in experience buffer. Filling...\n",
            "229 samples in experience buffer. Filling...\n",
            "331 samples in experience buffer. Filling...\n",
            "402 samples in experience buffer. Filling...\n",
            "469 samples in experience buffer. Filling...\n",
            "546 samples in experience buffer. Filling...\n",
            "679 samples in experience buffer. Filling...\n",
            "747 samples in experience buffer. Filling...\n",
            "861 samples in experience buffer. Filling...\n",
            "1005 samples in experience buffer. Filling...\n",
            "1130 samples in experience buffer. Filling...\n",
            "1238 samples in experience buffer. Filling...\n",
            "1330 samples in experience buffer. Filling...\n",
            "1455 samples in experience buffer. Filling...\n",
            "1537 samples in experience buffer. Filling...\n",
            "1637 samples in experience buffer. Filling...\n",
            "1738 samples in experience buffer. Filling...\n",
            "1835 samples in experience buffer. Filling...\n",
            "1898 samples in experience buffer. Filling...\n",
            "1982 samples in experience buffer. Filling...\n",
            "2061 samples in experience buffer. Filling...\n",
            "2139 samples in experience buffer. Filling...\n",
            "2228 samples in experience buffer. Filling...\n",
            "2322 samples in experience buffer. Filling...\n",
            "2471 samples in experience buffer. Filling...\n",
            "2552 samples in experience buffer. Filling...\n",
            "2688 samples in experience buffer. Filling...\n",
            "2782 samples in experience buffer. Filling...\n",
            "2852 samples in experience buffer. Filling...\n",
            "2924 samples in experience buffer. Filling...\n",
            "3058 samples in experience buffer. Filling...\n",
            "3141 samples in experience buffer. Filling...\n",
            "3207 samples in experience buffer. Filling...\n",
            "3275 samples in experience buffer. Filling...\n",
            "3371 samples in experience buffer. Filling...\n",
            "3435 samples in experience buffer. Filling...\n",
            "3540 samples in experience buffer. Filling...\n",
            "3656 samples in experience buffer. Filling...\n",
            "3727 samples in experience buffer. Filling...\n",
            "3865 samples in experience buffer. Filling...\n",
            "3959 samples in experience buffer. Filling...\n",
            "4096 samples in experience buffer. Filling...\n",
            "4191 samples in experience buffer. Filling...\n",
            "4279 samples in experience buffer. Filling...\n",
            "4346 samples in experience buffer. Filling...\n",
            "4444 samples in experience buffer. Filling...\n",
            "4551 samples in experience buffer. Filling...\n",
            "4642 samples in experience buffer. Filling...\n",
            "4749 samples in experience buffer. Filling...\n",
            "4820 samples in experience buffer. Filling...\n",
            "4912 samples in experience buffer. Filling...\n",
            "5043 samples in experience buffer. Filling...\n",
            "5132 samples in experience buffer. Filling...\n",
            "5233 samples in experience buffer. Filling...\n",
            "5297 samples in experience buffer. Filling...\n",
            "5363 samples in experience buffer. Filling...\n",
            "5464 samples in experience buffer. Filling...\n",
            "5544 samples in experience buffer. Filling...\n",
            "5652 samples in experience buffer. Filling...\n",
            "5715 samples in experience buffer. Filling...\n",
            "5802 samples in experience buffer. Filling...\n",
            "5934 samples in experience buffer. Filling...\n",
            "6008 samples in experience buffer. Filling...\n",
            "6084 samples in experience buffer. Filling...\n",
            "6206 samples in experience buffer. Filling...\n",
            "6317 samples in experience buffer. Filling...\n",
            "6398 samples in experience buffer. Filling...\n",
            "6517 samples in experience buffer. Filling...\n",
            "6611 samples in experience buffer. Filling...\n",
            "6678 samples in experience buffer. Filling...\n",
            "6784 samples in experience buffer. Filling...\n",
            "6865 samples in experience buffer. Filling...\n",
            "6960 samples in experience buffer. Filling...\n",
            "7034 samples in experience buffer. Filling...\n",
            "7127 samples in experience buffer. Filling...\n",
            "7222 samples in experience buffer. Filling...\n",
            "7351 samples in experience buffer. Filling...\n",
            "7438 samples in experience buffer. Filling...\n",
            "7569 samples in experience buffer. Filling...\n",
            "7657 samples in experience buffer. Filling...\n",
            "7785 samples in experience buffer. Filling...\n",
            "7908 samples in experience buffer. Filling...\n",
            "8000 samples in experience buffer. Filling...\n",
            "8077 samples in experience buffer. Filling...\n",
            "8157 samples in experience buffer. Filling...\n",
            "8238 samples in experience buffer. Filling...\n",
            "8306 samples in experience buffer. Filling...\n",
            "8409 samples in experience buffer. Filling...\n",
            "8510 samples in experience buffer. Filling...\n",
            "8611 samples in experience buffer. Filling...\n",
            "8712 samples in experience buffer. Filling...\n",
            "8773 samples in experience buffer. Filling...\n",
            "8844 samples in experience buffer. Filling...\n",
            "8945 samples in experience buffer. Filling...\n",
            "9043 samples in experience buffer. Filling...\n",
            "9101 samples in experience buffer. Filling...\n",
            "9176 samples in experience buffer. Filling...\n",
            "9282 samples in experience buffer. Filling...\n",
            "9338 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "9436 samples in experience buffer. Filling...\n",
            "9537 samples in experience buffer. Filling...\n",
            "9645 samples in experience buffer. Filling...\n",
            "9730 samples in experience buffer. Filling...\n",
            "9812 samples in experience buffer. Filling...\n",
            "9937 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 5.0 K  | train\n",
            "1 | target_q_net | DQN  | 5.0 K  | train\n",
            "----------------------------------------------\n",
            "10.0 K    Trainable params\n",
            "0         Non-trainable params\n",
            "10.0 K    Total params\n",
            "0.040     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c0e897368aaa44bcbcf3c50da6786de3",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 08:29:21,128] Trial 5 finished with value: -28.610305786132812 and parameters: {'lr': 0.0601675714960212, 'gamma': 0.18866281424771947, 'hidden_size': 64, 'eps_end': 0.10430826402327686}. Best is trial 5 with value: -28.610305786132812.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "73 samples in experience buffer. Filling...\n",
            "158 samples in experience buffer. Filling...\n",
            "228 samples in experience buffer. Filling...\n",
            "315 samples in experience buffer. Filling...\n",
            "432 samples in experience buffer. Filling...\n",
            "553 samples in experience buffer. Filling...\n",
            "687 samples in experience buffer. Filling...\n",
            "785 samples in experience buffer. Filling...\n",
            "891 samples in experience buffer. Filling...\n",
            "969 samples in experience buffer. Filling...\n",
            "1051 samples in experience buffer. Filling...\n",
            "1140 samples in experience buffer. Filling...\n",
            "1206 samples in experience buffer. Filling...\n",
            "1281 samples in experience buffer. Filling...\n",
            "1395 samples in experience buffer. Filling...\n",
            "1482 samples in experience buffer. Filling...\n",
            "1572 samples in experience buffer. Filling...\n",
            "1633 samples in experience buffer. Filling...\n",
            "1697 samples in experience buffer. Filling...\n",
            "1761 samples in experience buffer. Filling...\n",
            "1867 samples in experience buffer. Filling...\n",
            "1950 samples in experience buffer. Filling...\n",
            "2016 samples in experience buffer. Filling...\n",
            "2112 samples in experience buffer. Filling...\n",
            "2195 samples in experience buffer. Filling...\n",
            "2284 samples in experience buffer. Filling...\n",
            "2355 samples in experience buffer. Filling...\n",
            "2494 samples in experience buffer. Filling...\n",
            "2560 samples in experience buffer. Filling...\n",
            "2656 samples in experience buffer. Filling...\n",
            "2722 samples in experience buffer. Filling...\n",
            "2834 samples in experience buffer. Filling...\n",
            "2899 samples in experience buffer. Filling...\n",
            "2992 samples in experience buffer. Filling...\n",
            "3090 samples in experience buffer. Filling...\n",
            "3187 samples in experience buffer. Filling...\n",
            "3292 samples in experience buffer. Filling...\n",
            "3395 samples in experience buffer. Filling...\n",
            "3483 samples in experience buffer. Filling...\n",
            "3600 samples in experience buffer. Filling...\n",
            "3672 samples in experience buffer. Filling...\n",
            "3741 samples in experience buffer. Filling...\n",
            "3834 samples in experience buffer. Filling...\n",
            "3955 samples in experience buffer. Filling...\n",
            "4045 samples in experience buffer. Filling...\n",
            "4149 samples in experience buffer. Filling...\n",
            "4240 samples in experience buffer. Filling...\n",
            "4321 samples in experience buffer. Filling...\n",
            "4430 samples in experience buffer. Filling...\n",
            "4517 samples in experience buffer. Filling...\n",
            "4598 samples in experience buffer. Filling...\n",
            "4689 samples in experience buffer. Filling...\n",
            "4765 samples in experience buffer. Filling...\n",
            "4903 samples in experience buffer. Filling...\n",
            "4982 samples in experience buffer. Filling...\n",
            "5088 samples in experience buffer. Filling...\n",
            "5152 samples in experience buffer. Filling...\n",
            "5252 samples in experience buffer. Filling...\n",
            "5337 samples in experience buffer. Filling...\n",
            "5452 samples in experience buffer. Filling...\n",
            "5555 samples in experience buffer. Filling...\n",
            "5663 samples in experience buffer. Filling...\n",
            "5793 samples in experience buffer. Filling...\n",
            "5890 samples in experience buffer. Filling...\n",
            "5986 samples in experience buffer. Filling...\n",
            "6104 samples in experience buffer. Filling...\n",
            "6179 samples in experience buffer. Filling...\n",
            "6296 samples in experience buffer. Filling...\n",
            "6371 samples in experience buffer. Filling...\n",
            "6450 samples in experience buffer. Filling...\n",
            "6540 samples in experience buffer. Filling...\n",
            "6673 samples in experience buffer. Filling...\n",
            "6745 samples in experience buffer. Filling...\n",
            "6872 samples in experience buffer. Filling...\n",
            "6962 samples in experience buffer. Filling...\n",
            "7080 samples in experience buffer. Filling...\n",
            "7155 samples in experience buffer. Filling...\n",
            "7252 samples in experience buffer. Filling...\n",
            "7344 samples in experience buffer. Filling...\n",
            "7444 samples in experience buffer. Filling...\n",
            "7530 samples in experience buffer. Filling...\n",
            "7609 samples in experience buffer. Filling...\n",
            "7672 samples in experience buffer. Filling...\n",
            "7744 samples in experience buffer. Filling...\n",
            "7844 samples in experience buffer. Filling...\n",
            "7916 samples in experience buffer. Filling...\n",
            "8008 samples in experience buffer. Filling...\n",
            "8091 samples in experience buffer. Filling...\n",
            "8173 samples in experience buffer. Filling...\n",
            "8272 samples in experience buffer. Filling...\n",
            "8384 samples in experience buffer. Filling...\n",
            "8520 samples in experience buffer. Filling...\n",
            "8622 samples in experience buffer. Filling...\n",
            "8722 samples in experience buffer. Filling...\n",
            "8846 samples in experience buffer. Filling...\n",
            "8959 samples in experience buffer. Filling...\n",
            "9018 samples in experience buffer. Filling...\n",
            "9101 samples in experience buffer. Filling...\n",
            "9168 samples in experience buffer. Filling...\n",
            "9227 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "9334 samples in experience buffer. Filling...\n",
            "9421 samples in experience buffer. Filling...\n",
            "9524 samples in experience buffer. Filling...\n",
            "9623 samples in experience buffer. Filling...\n",
            "9681 samples in experience buffer. Filling...\n",
            "9762 samples in experience buffer. Filling...\n",
            "9823 samples in experience buffer. Filling...\n",
            "9910 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n",
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 18.2 K | train\n",
            "1 | target_q_net | DQN  | 18.2 K | train\n",
            "----------------------------------------------\n",
            "36.4 K    Trainable params\n",
            "0         Non-trainable params\n",
            "36.4 K    Total params\n",
            "0.145     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8c1a18ff3e8b4e8a97e023d8c47695ff",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 08:32:04,041] Trial 6 finished with value: -9.713607788085938 and parameters: {'lr': 0.00010628792889415254, 'gamma': 0.34208771766571366, 'hidden_size': 128, 'eps_end': 0.2799671764234091}. Best is trial 6 with value: -9.713607788085938.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "73 samples in experience buffer. Filling...\n",
            "147 samples in experience buffer. Filling...\n",
            "226 samples in experience buffer. Filling...\n",
            "319 samples in experience buffer. Filling...\n",
            "402 samples in experience buffer. Filling...\n",
            "477 samples in experience buffer. Filling...\n",
            "558 samples in experience buffer. Filling...\n",
            "647 samples in experience buffer. Filling...\n",
            "743 samples in experience buffer. Filling...\n",
            "828 samples in experience buffer. Filling...\n",
            "923 samples in experience buffer. Filling...\n",
            "993 samples in experience buffer. Filling...\n",
            "1069 samples in experience buffer. Filling...\n",
            "1141 samples in experience buffer. Filling...\n",
            "1214 samples in experience buffer. Filling...\n",
            "1286 samples in experience buffer. Filling...\n",
            "1357 samples in experience buffer. Filling...\n",
            "1437 samples in experience buffer. Filling...\n",
            "1533 samples in experience buffer. Filling...\n",
            "1596 samples in experience buffer. Filling...\n",
            "1710 samples in experience buffer. Filling...\n",
            "1804 samples in experience buffer. Filling...\n",
            "1871 samples in experience buffer. Filling...\n",
            "1977 samples in experience buffer. Filling...\n",
            "2082 samples in experience buffer. Filling...\n",
            "2189 samples in experience buffer. Filling...\n",
            "2271 samples in experience buffer. Filling...\n",
            "2399 samples in experience buffer. Filling...\n",
            "2475 samples in experience buffer. Filling...\n",
            "2577 samples in experience buffer. Filling...\n",
            "2676 samples in experience buffer. Filling...\n",
            "2760 samples in experience buffer. Filling...\n",
            "2835 samples in experience buffer. Filling...\n",
            "2923 samples in experience buffer. Filling...\n",
            "3041 samples in experience buffer. Filling...\n",
            "3113 samples in experience buffer. Filling...\n",
            "3229 samples in experience buffer. Filling...\n",
            "3318 samples in experience buffer. Filling...\n",
            "3428 samples in experience buffer. Filling...\n",
            "3523 samples in experience buffer. Filling...\n",
            "3622 samples in experience buffer. Filling...\n",
            "3728 samples in experience buffer. Filling...\n",
            "3789 samples in experience buffer. Filling...\n",
            "3875 samples in experience buffer. Filling...\n",
            "3973 samples in experience buffer. Filling...\n",
            "4097 samples in experience buffer. Filling...\n",
            "4186 samples in experience buffer. Filling...\n",
            "4307 samples in experience buffer. Filling...\n",
            "4421 samples in experience buffer. Filling...\n",
            "4516 samples in experience buffer. Filling...\n",
            "4589 samples in experience buffer. Filling...\n",
            "4689 samples in experience buffer. Filling...\n",
            "4763 samples in experience buffer. Filling...\n",
            "4875 samples in experience buffer. Filling...\n",
            "4956 samples in experience buffer. Filling...\n",
            "5016 samples in experience buffer. Filling...\n",
            "5131 samples in experience buffer. Filling...\n",
            "5239 samples in experience buffer. Filling...\n",
            "5371 samples in experience buffer. Filling...\n",
            "5462 samples in experience buffer. Filling...\n",
            "5576 samples in experience buffer. Filling...\n",
            "5691 samples in experience buffer. Filling...\n",
            "5768 samples in experience buffer. Filling...\n",
            "5877 samples in experience buffer. Filling...\n",
            "6016 samples in experience buffer. Filling...\n",
            "6146 samples in experience buffer. Filling...\n",
            "6234 samples in experience buffer. Filling...\n",
            "6321 samples in experience buffer. Filling...\n",
            "6416 samples in experience buffer. Filling...\n",
            "6495 samples in experience buffer. Filling...\n",
            "6571 samples in experience buffer. Filling...\n",
            "6646 samples in experience buffer. Filling...\n",
            "6746 samples in experience buffer. Filling...\n",
            "6849 samples in experience buffer. Filling...\n",
            "6919 samples in experience buffer. Filling...\n",
            "6983 samples in experience buffer. Filling...\n",
            "7061 samples in experience buffer. Filling...\n",
            "7240 samples in experience buffer. Filling...\n",
            "7309 samples in experience buffer. Filling...\n",
            "7407 samples in experience buffer. Filling...\n",
            "7505 samples in experience buffer. Filling...\n",
            "7622 samples in experience buffer. Filling...\n",
            "7727 samples in experience buffer. Filling...\n",
            "7854 samples in experience buffer. Filling...\n",
            "7924 samples in experience buffer. Filling...\n",
            "8027 samples in experience buffer. Filling...\n",
            "8124 samples in experience buffer. Filling...\n",
            "8198 samples in experience buffer. Filling...\n",
            "8284 samples in experience buffer. Filling...\n",
            "8373 samples in experience buffer. Filling...\n",
            "8460 samples in experience buffer. Filling...\n",
            "8534 samples in experience buffer. Filling...\n",
            "8653 samples in experience buffer. Filling...\n",
            "8771 samples in experience buffer. Filling...\n",
            "8836 samples in experience buffer. Filling...\n",
            "8938 samples in experience buffer. Filling...\n",
            "9015 samples in experience buffer. Filling...\n",
            "9124 samples in experience buffer. Filling...\n",
            "9228 samples in experience buffer. Filling...\n",
            "9298 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "9406 samples in experience buffer. Filling...\n",
            "9491 samples in experience buffer. Filling...\n",
            "9575 samples in experience buffer. Filling...\n",
            "9649 samples in experience buffer. Filling...\n",
            "9736 samples in experience buffer. Filling...\n",
            "9825 samples in experience buffer. Filling...\n",
            "9908 samples in experience buffer. Filling...\n",
            "9988 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 18.2 K | train\n",
            "1 | target_q_net | DQN  | 18.2 K | train\n",
            "----------------------------------------------\n",
            "36.4 K    Trainable params\n",
            "0         Non-trainable params\n",
            "36.4 K    Total params\n",
            "0.145     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "96db4b1282894ed98e46f8f9c4cb03e3",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 08:35:24,447] Trial 7 finished with value: -196.81130981445312 and parameters: {'lr': 0.00026026619042916895, 'gamma': 0.48168220788305627, 'hidden_size': 128, 'eps_end': 0.043055372591207175}. Best is trial 6 with value: -9.713607788085938.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "93 samples in experience buffer. Filling...\n",
            "184 samples in experience buffer. Filling...\n",
            "282 samples in experience buffer. Filling...\n",
            "371 samples in experience buffer. Filling...\n",
            "445 samples in experience buffer. Filling...\n",
            "538 samples in experience buffer. Filling...\n",
            "601 samples in experience buffer. Filling...\n",
            "670 samples in experience buffer. Filling...\n",
            "750 samples in experience buffer. Filling...\n",
            "856 samples in experience buffer. Filling...\n",
            "943 samples in experience buffer. Filling...\n",
            "1007 samples in experience buffer. Filling...\n",
            "1114 samples in experience buffer. Filling...\n",
            "1189 samples in experience buffer. Filling...\n",
            "1262 samples in experience buffer. Filling...\n",
            "1334 samples in experience buffer. Filling...\n",
            "1422 samples in experience buffer. Filling...\n",
            "1516 samples in experience buffer. Filling...\n",
            "1625 samples in experience buffer. Filling...\n",
            "1733 samples in experience buffer. Filling...\n",
            "1843 samples in experience buffer. Filling...\n",
            "1939 samples in experience buffer. Filling...\n",
            "2032 samples in experience buffer. Filling...\n",
            "2102 samples in experience buffer. Filling...\n",
            "2202 samples in experience buffer. Filling...\n",
            "2307 samples in experience buffer. Filling...\n",
            "2385 samples in experience buffer. Filling...\n",
            "2488 samples in experience buffer. Filling...\n",
            "2578 samples in experience buffer. Filling...\n",
            "2672 samples in experience buffer. Filling...\n",
            "2744 samples in experience buffer. Filling...\n",
            "2825 samples in experience buffer. Filling...\n",
            "2924 samples in experience buffer. Filling...\n",
            "3018 samples in experience buffer. Filling...\n",
            "3117 samples in experience buffer. Filling...\n",
            "3196 samples in experience buffer. Filling...\n",
            "3273 samples in experience buffer. Filling...\n",
            "3372 samples in experience buffer. Filling...\n",
            "3436 samples in experience buffer. Filling...\n",
            "3516 samples in experience buffer. Filling...\n",
            "3633 samples in experience buffer. Filling...\n",
            "3734 samples in experience buffer. Filling...\n",
            "3824 samples in experience buffer. Filling...\n",
            "3902 samples in experience buffer. Filling...\n",
            "3991 samples in experience buffer. Filling...\n",
            "4093 samples in experience buffer. Filling...\n",
            "4206 samples in experience buffer. Filling...\n",
            "4334 samples in experience buffer. Filling...\n",
            "4400 samples in experience buffer. Filling...\n",
            "4512 samples in experience buffer. Filling...\n",
            "4623 samples in experience buffer. Filling...\n",
            "4740 samples in experience buffer. Filling...\n",
            "4833 samples in experience buffer. Filling...\n",
            "4906 samples in experience buffer. Filling...\n",
            "5013 samples in experience buffer. Filling...\n",
            "5096 samples in experience buffer. Filling...\n",
            "5225 samples in experience buffer. Filling...\n",
            "5307 samples in experience buffer. Filling...\n",
            "5439 samples in experience buffer. Filling...\n",
            "5519 samples in experience buffer. Filling...\n",
            "5611 samples in experience buffer. Filling...\n",
            "5681 samples in experience buffer. Filling...\n",
            "5754 samples in experience buffer. Filling...\n",
            "5845 samples in experience buffer. Filling...\n",
            "5937 samples in experience buffer. Filling...\n",
            "6005 samples in experience buffer. Filling...\n",
            "6081 samples in experience buffer. Filling...\n",
            "6154 samples in experience buffer. Filling...\n",
            "6292 samples in experience buffer. Filling...\n",
            "6357 samples in experience buffer. Filling...\n",
            "6446 samples in experience buffer. Filling...\n",
            "6550 samples in experience buffer. Filling...\n",
            "6659 samples in experience buffer. Filling...\n",
            "6779 samples in experience buffer. Filling...\n",
            "6878 samples in experience buffer. Filling...\n",
            "6983 samples in experience buffer. Filling...\n",
            "7124 samples in experience buffer. Filling...\n",
            "7224 samples in experience buffer. Filling...\n",
            "7299 samples in experience buffer. Filling...\n",
            "7393 samples in experience buffer. Filling...\n",
            "7455 samples in experience buffer. Filling...\n",
            "7518 samples in experience buffer. Filling...\n",
            "7635 samples in experience buffer. Filling...\n",
            "7718 samples in experience buffer. Filling...\n",
            "7884 samples in experience buffer. Filling...\n",
            "7958 samples in experience buffer. Filling...\n",
            "8086 samples in experience buffer. Filling...\n",
            "8167 samples in experience buffer. Filling...\n",
            "8243 samples in experience buffer. Filling...\n",
            "8311 samples in experience buffer. Filling...\n",
            "8406 samples in experience buffer. Filling...\n",
            "8479 samples in experience buffer. Filling...\n",
            "8598 samples in experience buffer. Filling...\n",
            "8691 samples in experience buffer. Filling...\n",
            "8781 samples in experience buffer. Filling...\n",
            "8858 samples in experience buffer. Filling...\n",
            "8922 samples in experience buffer. Filling...\n",
            "9003 samples in experience buffer. Filling...\n",
            "9122 samples in experience buffer. Filling...\n",
            "9219 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "9352 samples in experience buffer. Filling...\n",
            "9431 samples in experience buffer. Filling...\n",
            "9519 samples in experience buffer. Filling...\n",
            "9613 samples in experience buffer. Filling...\n",
            "9679 samples in experience buffer. Filling...\n",
            "9742 samples in experience buffer. Filling...\n",
            "9802 samples in experience buffer. Filling...\n",
            "9903 samples in experience buffer. Filling...\n",
            "9975 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 1.5 K  | train\n",
            "1 | target_q_net | DQN  | 1.5 K  | train\n",
            "----------------------------------------------\n",
            "3.0 K     Trainable params\n",
            "0         Non-trainable params\n",
            "3.0 K     Total params\n",
            "0.012     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "53d7821e4e4640d7ad9849464acc0920",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 08:38:30,332] Trial 8 finished with value: -45.47260665893555 and parameters: {'lr': 0.0015243352982419786, 'gamma': 0.18713698596785566, 'hidden_size': 32, 'eps_end': 0.20775113729534253}. Best is trial 6 with value: -9.713607788085938.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "89 samples in experience buffer. Filling...\n",
            "184 samples in experience buffer. Filling...\n",
            "280 samples in experience buffer. Filling...\n",
            "379 samples in experience buffer. Filling...\n",
            "461 samples in experience buffer. Filling...\n",
            "552 samples in experience buffer. Filling...\n",
            "630 samples in experience buffer. Filling...\n",
            "694 samples in experience buffer. Filling...\n",
            "795 samples in experience buffer. Filling...\n",
            "896 samples in experience buffer. Filling...\n",
            "990 samples in experience buffer. Filling...\n",
            "1089 samples in experience buffer. Filling...\n",
            "1153 samples in experience buffer. Filling...\n",
            "1288 samples in experience buffer. Filling...\n",
            "1398 samples in experience buffer. Filling...\n",
            "1456 samples in experience buffer. Filling...\n",
            "1575 samples in experience buffer. Filling...\n",
            "1631 samples in experience buffer. Filling...\n",
            "1697 samples in experience buffer. Filling...\n",
            "1795 samples in experience buffer. Filling...\n",
            "1859 samples in experience buffer. Filling...\n",
            "1948 samples in experience buffer. Filling...\n",
            "2052 samples in experience buffer. Filling...\n",
            "2117 samples in experience buffer. Filling...\n",
            "2213 samples in experience buffer. Filling...\n",
            "2349 samples in experience buffer. Filling...\n",
            "2475 samples in experience buffer. Filling...\n",
            "2596 samples in experience buffer. Filling...\n",
            "2669 samples in experience buffer. Filling...\n",
            "2742 samples in experience buffer. Filling...\n",
            "2805 samples in experience buffer. Filling...\n",
            "2886 samples in experience buffer. Filling...\n",
            "2970 samples in experience buffer. Filling...\n",
            "3074 samples in experience buffer. Filling...\n",
            "3167 samples in experience buffer. Filling...\n",
            "3303 samples in experience buffer. Filling...\n",
            "3394 samples in experience buffer. Filling...\n",
            "3464 samples in experience buffer. Filling...\n",
            "3546 samples in experience buffer. Filling...\n",
            "3619 samples in experience buffer. Filling...\n",
            "3687 samples in experience buffer. Filling...\n",
            "3750 samples in experience buffer. Filling...\n",
            "3853 samples in experience buffer. Filling...\n",
            "3919 samples in experience buffer. Filling...\n",
            "4032 samples in experience buffer. Filling...\n",
            "4140 samples in experience buffer. Filling...\n",
            "4246 samples in experience buffer. Filling...\n",
            "4331 samples in experience buffer. Filling...\n",
            "4423 samples in experience buffer. Filling...\n",
            "4503 samples in experience buffer. Filling...\n",
            "4643 samples in experience buffer. Filling...\n",
            "4736 samples in experience buffer. Filling...\n",
            "4852 samples in experience buffer. Filling...\n",
            "4934 samples in experience buffer. Filling...\n",
            "5042 samples in experience buffer. Filling...\n",
            "5115 samples in experience buffer. Filling...\n",
            "5264 samples in experience buffer. Filling...\n",
            "5324 samples in experience buffer. Filling...\n",
            "5386 samples in experience buffer. Filling...\n",
            "5471 samples in experience buffer. Filling...\n",
            "5534 samples in experience buffer. Filling...\n",
            "5639 samples in experience buffer. Filling...\n",
            "5720 samples in experience buffer. Filling...\n",
            "5802 samples in experience buffer. Filling...\n",
            "5869 samples in experience buffer. Filling...\n",
            "5983 samples in experience buffer. Filling...\n",
            "6098 samples in experience buffer. Filling...\n",
            "6186 samples in experience buffer. Filling...\n",
            "6263 samples in experience buffer. Filling...\n",
            "6379 samples in experience buffer. Filling...\n",
            "6471 samples in experience buffer. Filling...\n",
            "6545 samples in experience buffer. Filling...\n",
            "6657 samples in experience buffer. Filling...\n",
            "6788 samples in experience buffer. Filling...\n",
            "6896 samples in experience buffer. Filling...\n",
            "7002 samples in experience buffer. Filling...\n",
            "7093 samples in experience buffer. Filling...\n",
            "7176 samples in experience buffer. Filling...\n",
            "7248 samples in experience buffer. Filling...\n",
            "7363 samples in experience buffer. Filling...\n",
            "7477 samples in experience buffer. Filling...\n",
            "7571 samples in experience buffer. Filling...\n",
            "7644 samples in experience buffer. Filling...\n",
            "7721 samples in experience buffer. Filling...\n",
            "7847 samples in experience buffer. Filling...\n",
            "7942 samples in experience buffer. Filling...\n",
            "8004 samples in experience buffer. Filling...\n",
            "8080 samples in experience buffer. Filling...\n",
            "8181 samples in experience buffer. Filling...\n",
            "8253 samples in experience buffer. Filling...\n",
            "8331 samples in experience buffer. Filling...\n",
            "8409 samples in experience buffer. Filling...\n",
            "8540 samples in experience buffer. Filling...\n",
            "8639 samples in experience buffer. Filling...\n",
            "8701 samples in experience buffer. Filling...\n",
            "8800 samples in experience buffer. Filling...\n",
            "8908 samples in experience buffer. Filling...\n",
            "8990 samples in experience buffer. Filling...\n",
            "9097 samples in experience buffer. Filling...\n",
            "9208 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "9306 samples in experience buffer. Filling...\n",
            "9425 samples in experience buffer. Filling...\n",
            "9493 samples in experience buffer. Filling...\n",
            "9612 samples in experience buffer. Filling...\n",
            "9729 samples in experience buffer. Filling...\n",
            "9798 samples in experience buffer. Filling...\n",
            "9901 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 18.2 K | train\n",
            "1 | target_q_net | DQN  | 18.2 K | train\n",
            "----------------------------------------------\n",
            "36.4 K    Trainable params\n",
            "0         Non-trainable params\n",
            "36.4 K    Total params\n",
            "0.145     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0717ff004a7a4ba69e5ccae6ad0d72e6",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 08:41:30,348] Trial 9 finished with value: 252.13320922851562 and parameters: {'lr': 0.0036502063949420765, 'gamma': 0.6207765698639972, 'hidden_size': 128, 'eps_end': 0.08699195823968492}. Best is trial 9 with value: 252.13320922851562.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "65 samples in experience buffer. Filling...\n",
            "127 samples in experience buffer. Filling...\n",
            "223 samples in experience buffer. Filling...\n",
            "316 samples in experience buffer. Filling...\n",
            "451 samples in experience buffer. Filling...\n",
            "576 samples in experience buffer. Filling...\n",
            "630 samples in experience buffer. Filling...\n",
            "713 samples in experience buffer. Filling...\n",
            "792 samples in experience buffer. Filling...\n",
            "898 samples in experience buffer. Filling...\n",
            "999 samples in experience buffer. Filling...\n",
            "1136 samples in experience buffer. Filling...\n",
            "1252 samples in experience buffer. Filling...\n",
            "1323 samples in experience buffer. Filling...\n",
            "1450 samples in experience buffer. Filling...\n",
            "1548 samples in experience buffer. Filling...\n",
            "1609 samples in experience buffer. Filling...\n",
            "1704 samples in experience buffer. Filling...\n",
            "1785 samples in experience buffer. Filling...\n",
            "1887 samples in experience buffer. Filling...\n",
            "2019 samples in experience buffer. Filling...\n",
            "2108 samples in experience buffer. Filling...\n",
            "2227 samples in experience buffer. Filling...\n",
            "2341 samples in experience buffer. Filling...\n",
            "2439 samples in experience buffer. Filling...\n",
            "2540 samples in experience buffer. Filling...\n",
            "2605 samples in experience buffer. Filling...\n",
            "2683 samples in experience buffer. Filling...\n",
            "2796 samples in experience buffer. Filling...\n",
            "2888 samples in experience buffer. Filling...\n",
            "2984 samples in experience buffer. Filling...\n",
            "3073 samples in experience buffer. Filling...\n",
            "3135 samples in experience buffer. Filling...\n",
            "3239 samples in experience buffer. Filling...\n",
            "3367 samples in experience buffer. Filling...\n",
            "3462 samples in experience buffer. Filling...\n",
            "3578 samples in experience buffer. Filling...\n",
            "3686 samples in experience buffer. Filling...\n",
            "3795 samples in experience buffer. Filling...\n",
            "3866 samples in experience buffer. Filling...\n",
            "3935 samples in experience buffer. Filling...\n",
            "4062 samples in experience buffer. Filling...\n",
            "4132 samples in experience buffer. Filling...\n",
            "4205 samples in experience buffer. Filling...\n",
            "4343 samples in experience buffer. Filling...\n",
            "4446 samples in experience buffer. Filling...\n",
            "4552 samples in experience buffer. Filling...\n",
            "4657 samples in experience buffer. Filling...\n",
            "4763 samples in experience buffer. Filling...\n",
            "4871 samples in experience buffer. Filling...\n",
            "4970 samples in experience buffer. Filling...\n",
            "5071 samples in experience buffer. Filling...\n",
            "5159 samples in experience buffer. Filling...\n",
            "5247 samples in experience buffer. Filling...\n",
            "5356 samples in experience buffer. Filling...\n",
            "5464 samples in experience buffer. Filling...\n",
            "5570 samples in experience buffer. Filling...\n",
            "5655 samples in experience buffer. Filling...\n",
            "5758 samples in experience buffer. Filling...\n",
            "5839 samples in experience buffer. Filling...\n",
            "5903 samples in experience buffer. Filling...\n",
            "5993 samples in experience buffer. Filling...\n",
            "6090 samples in experience buffer. Filling...\n",
            "6155 samples in experience buffer. Filling...\n",
            "6247 samples in experience buffer. Filling...\n",
            "6329 samples in experience buffer. Filling...\n",
            "6443 samples in experience buffer. Filling...\n",
            "6565 samples in experience buffer. Filling...\n",
            "6657 samples in experience buffer. Filling...\n",
            "6777 samples in experience buffer. Filling...\n",
            "6856 samples in experience buffer. Filling...\n",
            "6945 samples in experience buffer. Filling...\n",
            "7043 samples in experience buffer. Filling...\n",
            "7137 samples in experience buffer. Filling...\n",
            "7242 samples in experience buffer. Filling...\n",
            "7302 samples in experience buffer. Filling...\n",
            "7373 samples in experience buffer. Filling...\n",
            "7474 samples in experience buffer. Filling...\n",
            "7582 samples in experience buffer. Filling...\n",
            "7644 samples in experience buffer. Filling...\n",
            "7716 samples in experience buffer. Filling...\n",
            "7812 samples in experience buffer. Filling...\n",
            "7906 samples in experience buffer. Filling...\n",
            "8043 samples in experience buffer. Filling...\n",
            "8127 samples in experience buffer. Filling...\n",
            "8204 samples in experience buffer. Filling...\n",
            "8319 samples in experience buffer. Filling...\n",
            "8411 samples in experience buffer. Filling...\n",
            "8511 samples in experience buffer. Filling...\n",
            "8620 samples in experience buffer. Filling...\n",
            "8731 samples in experience buffer. Filling...\n",
            "8814 samples in experience buffer. Filling...\n",
            "8912 samples in experience buffer. Filling...\n",
            "8989 samples in experience buffer. Filling...\n",
            "9063 samples in experience buffer. Filling...\n",
            "9143 samples in experience buffer. Filling...\n",
            "9219 samples in experience buffer. Filling...\n",
            "9322 samples in experience buffer. Filling...\n",
            "9436 samples in experience buffer. Filling...\n",
            "9526 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "9614 samples in experience buffer. Filling...\n",
            "9686 samples in experience buffer. Filling...\n",
            "9802 samples in experience buffer. Filling...\n",
            "9875 samples in experience buffer. Filling...\n",
            "9957 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 5.0 K  | train\n",
            "1 | target_q_net | DQN  | 5.0 K  | train\n",
            "----------------------------------------------\n",
            "10.0 K    Trainable params\n",
            "0         Non-trainable params\n",
            "10.0 K    Total params\n",
            "0.040     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4daf257456694518bf4ea076b39b8e31",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 08:44:52,633] Trial 10 finished with value: -174.33912658691406 and parameters: {'lr': 0.01967392966320188, 'gamma': 0.606155768840539, 'hidden_size': 64, 'eps_end': 0.006856685260701434}. Best is trial 9 with value: 252.13320922851562.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "104 samples in experience buffer. Filling...\n",
            "194 samples in experience buffer. Filling...\n",
            "312 samples in experience buffer. Filling...\n",
            "374 samples in experience buffer. Filling...\n",
            "446 samples in experience buffer. Filling...\n",
            "544 samples in experience buffer. Filling...\n",
            "615 samples in experience buffer. Filling...\n",
            "692 samples in experience buffer. Filling...\n",
            "822 samples in experience buffer. Filling...\n",
            "912 samples in experience buffer. Filling...\n",
            "982 samples in experience buffer. Filling...\n",
            "1078 samples in experience buffer. Filling...\n",
            "1159 samples in experience buffer. Filling...\n",
            "1236 samples in experience buffer. Filling...\n",
            "1347 samples in experience buffer. Filling...\n",
            "1439 samples in experience buffer. Filling...\n",
            "1560 samples in experience buffer. Filling...\n",
            "1652 samples in experience buffer. Filling...\n",
            "1738 samples in experience buffer. Filling...\n",
            "1799 samples in experience buffer. Filling...\n",
            "1912 samples in experience buffer. Filling...\n",
            "1978 samples in experience buffer. Filling...\n",
            "2073 samples in experience buffer. Filling...\n",
            "2178 samples in experience buffer. Filling...\n",
            "2276 samples in experience buffer. Filling...\n",
            "2385 samples in experience buffer. Filling...\n",
            "2466 samples in experience buffer. Filling...\n",
            "2552 samples in experience buffer. Filling...\n",
            "2667 samples in experience buffer. Filling...\n",
            "2746 samples in experience buffer. Filling...\n",
            "2829 samples in experience buffer. Filling...\n",
            "2898 samples in experience buffer. Filling...\n",
            "2989 samples in experience buffer. Filling...\n",
            "3048 samples in experience buffer. Filling...\n",
            "3133 samples in experience buffer. Filling...\n",
            "3272 samples in experience buffer. Filling...\n",
            "3359 samples in experience buffer. Filling...\n",
            "3435 samples in experience buffer. Filling...\n",
            "3533 samples in experience buffer. Filling...\n",
            "3605 samples in experience buffer. Filling...\n",
            "3678 samples in experience buffer. Filling...\n",
            "3755 samples in experience buffer. Filling...\n",
            "3872 samples in experience buffer. Filling...\n",
            "3935 samples in experience buffer. Filling...\n",
            "4018 samples in experience buffer. Filling...\n",
            "4087 samples in experience buffer. Filling...\n",
            "4160 samples in experience buffer. Filling...\n",
            "4263 samples in experience buffer. Filling...\n",
            "4341 samples in experience buffer. Filling...\n",
            "4450 samples in experience buffer. Filling...\n",
            "4519 samples in experience buffer. Filling...\n",
            "4585 samples in experience buffer. Filling...\n",
            "4702 samples in experience buffer. Filling...\n",
            "4771 samples in experience buffer. Filling...\n",
            "4870 samples in experience buffer. Filling...\n",
            "4998 samples in experience buffer. Filling...\n",
            "5078 samples in experience buffer. Filling...\n",
            "5140 samples in experience buffer. Filling...\n",
            "5270 samples in experience buffer. Filling...\n",
            "5345 samples in experience buffer. Filling...\n",
            "5439 samples in experience buffer. Filling...\n",
            "5576 samples in experience buffer. Filling...\n",
            "5648 samples in experience buffer. Filling...\n",
            "5723 samples in experience buffer. Filling...\n",
            "5816 samples in experience buffer. Filling...\n",
            "5908 samples in experience buffer. Filling...\n",
            "5986 samples in experience buffer. Filling...\n",
            "6072 samples in experience buffer. Filling...\n",
            "6135 samples in experience buffer. Filling...\n",
            "6217 samples in experience buffer. Filling...\n",
            "6333 samples in experience buffer. Filling...\n",
            "6429 samples in experience buffer. Filling...\n",
            "6547 samples in experience buffer. Filling...\n",
            "6621 samples in experience buffer. Filling...\n",
            "6694 samples in experience buffer. Filling...\n",
            "6793 samples in experience buffer. Filling...\n",
            "6882 samples in experience buffer. Filling...\n",
            "6978 samples in experience buffer. Filling...\n",
            "7058 samples in experience buffer. Filling...\n",
            "7124 samples in experience buffer. Filling...\n",
            "7249 samples in experience buffer. Filling...\n",
            "7334 samples in experience buffer. Filling...\n",
            "7408 samples in experience buffer. Filling...\n",
            "7510 samples in experience buffer. Filling...\n",
            "7602 samples in experience buffer. Filling...\n",
            "7693 samples in experience buffer. Filling...\n",
            "7770 samples in experience buffer. Filling...\n",
            "7895 samples in experience buffer. Filling...\n",
            "7955 samples in experience buffer. Filling...\n",
            "8024 samples in experience buffer. Filling...\n",
            "8110 samples in experience buffer. Filling...\n",
            "8189 samples in experience buffer. Filling...\n",
            "8311 samples in experience buffer. Filling...\n",
            "8418 samples in experience buffer. Filling...\n",
            "8525 samples in experience buffer. Filling...\n",
            "8612 samples in experience buffer. Filling...\n",
            "8685 samples in experience buffer. Filling...\n",
            "8763 samples in experience buffer. Filling...\n",
            "8862 samples in experience buffer. Filling...\n",
            "8987 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "9067 samples in experience buffer. Filling...\n",
            "9184 samples in experience buffer. Filling...\n",
            "9271 samples in experience buffer. Filling...\n",
            "9371 samples in experience buffer. Filling...\n",
            "9430 samples in experience buffer. Filling...\n",
            "9519 samples in experience buffer. Filling...\n",
            "9613 samples in experience buffer. Filling...\n",
            "9731 samples in experience buffer. Filling...\n",
            "9853 samples in experience buffer. Filling...\n",
            "9952 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n",
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 18.2 K | train\n",
            "1 | target_q_net | DQN  | 18.2 K | train\n",
            "----------------------------------------------\n",
            "36.4 K    Trainable params\n",
            "0         Non-trainable params\n",
            "36.4 K    Total params\n",
            "0.145     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "86e3eb08fcb54845adb02cb19e6feb00",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 08:47:40,095] Trial 11 finished with value: -46.776893615722656 and parameters: {'lr': 1.8064002272526133e-05, 'gamma': 0.07181682210634144, 'hidden_size': 128, 'eps_end': 0.28795300673604457}. Best is trial 9 with value: 252.13320922851562.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "134 samples in experience buffer. Filling...\n",
            "205 samples in experience buffer. Filling...\n",
            "283 samples in experience buffer. Filling...\n",
            "366 samples in experience buffer. Filling...\n",
            "490 samples in experience buffer. Filling...\n",
            "560 samples in experience buffer. Filling...\n",
            "639 samples in experience buffer. Filling...\n",
            "734 samples in experience buffer. Filling...\n",
            "835 samples in experience buffer. Filling...\n",
            "914 samples in experience buffer. Filling...\n",
            "979 samples in experience buffer. Filling...\n",
            "1090 samples in experience buffer. Filling...\n",
            "1201 samples in experience buffer. Filling...\n",
            "1300 samples in experience buffer. Filling...\n",
            "1396 samples in experience buffer. Filling...\n",
            "1515 samples in experience buffer. Filling...\n",
            "1574 samples in experience buffer. Filling...\n",
            "1681 samples in experience buffer. Filling...\n",
            "1775 samples in experience buffer. Filling...\n",
            "1867 samples in experience buffer. Filling...\n",
            "1958 samples in experience buffer. Filling...\n",
            "2064 samples in experience buffer. Filling...\n",
            "2166 samples in experience buffer. Filling...\n",
            "2292 samples in experience buffer. Filling...\n",
            "2356 samples in experience buffer. Filling...\n",
            "2454 samples in experience buffer. Filling...\n",
            "2531 samples in experience buffer. Filling...\n",
            "2630 samples in experience buffer. Filling...\n",
            "2698 samples in experience buffer. Filling...\n",
            "2788 samples in experience buffer. Filling...\n",
            "2859 samples in experience buffer. Filling...\n",
            "2994 samples in experience buffer. Filling...\n",
            "3097 samples in experience buffer. Filling...\n",
            "3169 samples in experience buffer. Filling...\n",
            "3287 samples in experience buffer. Filling...\n",
            "3349 samples in experience buffer. Filling...\n",
            "3421 samples in experience buffer. Filling...\n",
            "3525 samples in experience buffer. Filling...\n",
            "3643 samples in experience buffer. Filling...\n",
            "3749 samples in experience buffer. Filling...\n",
            "3879 samples in experience buffer. Filling...\n",
            "3949 samples in experience buffer. Filling...\n",
            "4030 samples in experience buffer. Filling...\n",
            "4127 samples in experience buffer. Filling...\n",
            "4191 samples in experience buffer. Filling...\n",
            "4295 samples in experience buffer. Filling...\n",
            "4381 samples in experience buffer. Filling...\n",
            "4490 samples in experience buffer. Filling...\n",
            "4585 samples in experience buffer. Filling...\n",
            "4649 samples in experience buffer. Filling...\n",
            "4710 samples in experience buffer. Filling...\n",
            "4813 samples in experience buffer. Filling...\n",
            "4891 samples in experience buffer. Filling...\n",
            "5008 samples in experience buffer. Filling...\n",
            "5106 samples in experience buffer. Filling...\n",
            "5186 samples in experience buffer. Filling...\n",
            "5261 samples in experience buffer. Filling...\n",
            "5346 samples in experience buffer. Filling...\n",
            "5479 samples in experience buffer. Filling...\n",
            "5604 samples in experience buffer. Filling...\n",
            "5743 samples in experience buffer. Filling...\n",
            "5862 samples in experience buffer. Filling...\n",
            "5968 samples in experience buffer. Filling...\n",
            "6047 samples in experience buffer. Filling...\n",
            "6114 samples in experience buffer. Filling...\n",
            "6238 samples in experience buffer. Filling...\n",
            "6321 samples in experience buffer. Filling...\n",
            "6446 samples in experience buffer. Filling...\n",
            "6549 samples in experience buffer. Filling...\n",
            "6669 samples in experience buffer. Filling...\n",
            "6761 samples in experience buffer. Filling...\n",
            "6866 samples in experience buffer. Filling...\n",
            "6998 samples in experience buffer. Filling...\n",
            "7122 samples in experience buffer. Filling...\n",
            "7190 samples in experience buffer. Filling...\n",
            "7314 samples in experience buffer. Filling...\n",
            "7397 samples in experience buffer. Filling...\n",
            "7494 samples in experience buffer. Filling...\n",
            "7602 samples in experience buffer. Filling...\n",
            "7690 samples in experience buffer. Filling...\n",
            "7768 samples in experience buffer. Filling...\n",
            "7850 samples in experience buffer. Filling...\n",
            "7955 samples in experience buffer. Filling...\n",
            "8052 samples in experience buffer. Filling...\n",
            "8132 samples in experience buffer. Filling...\n",
            "8197 samples in experience buffer. Filling...\n",
            "8279 samples in experience buffer. Filling...\n",
            "8343 samples in experience buffer. Filling...\n",
            "8428 samples in experience buffer. Filling...\n",
            "8503 samples in experience buffer. Filling...\n",
            "8557 samples in experience buffer. Filling...\n",
            "8618 samples in experience buffer. Filling...\n",
            "8690 samples in experience buffer. Filling...\n",
            "8788 samples in experience buffer. Filling...\n",
            "8854 samples in experience buffer. Filling...\n",
            "8932 samples in experience buffer. Filling...\n",
            "9073 samples in experience buffer. Filling...\n",
            "9175 samples in experience buffer. Filling...\n",
            "9265 samples in experience buffer. Filling...\n",
            "9386 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "9489 samples in experience buffer. Filling...\n",
            "9620 samples in experience buffer. Filling...\n",
            "9717 samples in experience buffer. Filling...\n",
            "9832 samples in experience buffer. Filling...\n",
            "9953 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 18.2 K | train\n",
            "1 | target_q_net | DQN  | 18.2 K | train\n",
            "----------------------------------------------\n",
            "36.4 K    Trainable params\n",
            "0         Non-trainable params\n",
            "36.4 K    Total params\n",
            "0.145     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "75b00b5688e4459692136234673ce055",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 08:50:59,104] Trial 12 finished with value: 142.99990844726562 and parameters: {'lr': 0.00010542725867673657, 'gamma': 0.49336346275537785, 'hidden_size': 128, 'eps_end': 0.15807533418906972}. Best is trial 9 with value: 252.13320922851562.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "71 samples in experience buffer. Filling...\n",
            "189 samples in experience buffer. Filling...\n",
            "273 samples in experience buffer. Filling...\n",
            "360 samples in experience buffer. Filling...\n",
            "452 samples in experience buffer. Filling...\n",
            "549 samples in experience buffer. Filling...\n",
            "656 samples in experience buffer. Filling...\n",
            "716 samples in experience buffer. Filling...\n",
            "841 samples in experience buffer. Filling...\n",
            "907 samples in experience buffer. Filling...\n",
            "990 samples in experience buffer. Filling...\n",
            "1060 samples in experience buffer. Filling...\n",
            "1163 samples in experience buffer. Filling...\n",
            "1263 samples in experience buffer. Filling...\n",
            "1333 samples in experience buffer. Filling...\n",
            "1428 samples in experience buffer. Filling...\n",
            "1520 samples in experience buffer. Filling...\n",
            "1664 samples in experience buffer. Filling...\n",
            "1763 samples in experience buffer. Filling...\n",
            "1850 samples in experience buffer. Filling...\n",
            "1933 samples in experience buffer. Filling...\n",
            "2008 samples in experience buffer. Filling...\n",
            "2103 samples in experience buffer. Filling...\n",
            "2196 samples in experience buffer. Filling...\n",
            "2301 samples in experience buffer. Filling...\n",
            "2380 samples in experience buffer. Filling...\n",
            "2446 samples in experience buffer. Filling...\n",
            "2573 samples in experience buffer. Filling...\n",
            "2688 samples in experience buffer. Filling...\n",
            "2778 samples in experience buffer. Filling...\n",
            "2864 samples in experience buffer. Filling...\n",
            "2952 samples in experience buffer. Filling...\n",
            "3077 samples in experience buffer. Filling...\n",
            "3165 samples in experience buffer. Filling...\n",
            "3260 samples in experience buffer. Filling...\n",
            "3331 samples in experience buffer. Filling...\n",
            "3438 samples in experience buffer. Filling...\n",
            "3526 samples in experience buffer. Filling...\n",
            "3598 samples in experience buffer. Filling...\n",
            "3678 samples in experience buffer. Filling...\n",
            "3817 samples in experience buffer. Filling...\n",
            "3886 samples in experience buffer. Filling...\n",
            "3983 samples in experience buffer. Filling...\n",
            "4044 samples in experience buffer. Filling...\n",
            "4146 samples in experience buffer. Filling...\n",
            "4253 samples in experience buffer. Filling...\n",
            "4367 samples in experience buffer. Filling...\n",
            "4457 samples in experience buffer. Filling...\n",
            "4536 samples in experience buffer. Filling...\n",
            "4606 samples in experience buffer. Filling...\n",
            "4676 samples in experience buffer. Filling...\n",
            "4736 samples in experience buffer. Filling...\n",
            "4841 samples in experience buffer. Filling...\n",
            "4938 samples in experience buffer. Filling...\n",
            "5017 samples in experience buffer. Filling...\n",
            "5148 samples in experience buffer. Filling...\n",
            "5243 samples in experience buffer. Filling...\n",
            "5308 samples in experience buffer. Filling...\n",
            "5386 samples in experience buffer. Filling...\n",
            "5450 samples in experience buffer. Filling...\n",
            "5527 samples in experience buffer. Filling...\n",
            "5629 samples in experience buffer. Filling...\n",
            "5721 samples in experience buffer. Filling...\n",
            "5783 samples in experience buffer. Filling...\n",
            "5866 samples in experience buffer. Filling...\n",
            "5944 samples in experience buffer. Filling...\n",
            "6033 samples in experience buffer. Filling...\n",
            "6117 samples in experience buffer. Filling...\n",
            "6220 samples in experience buffer. Filling...\n",
            "6332 samples in experience buffer. Filling...\n",
            "6402 samples in experience buffer. Filling...\n",
            "6513 samples in experience buffer. Filling...\n",
            "6596 samples in experience buffer. Filling...\n",
            "6666 samples in experience buffer. Filling...\n",
            "6798 samples in experience buffer. Filling...\n",
            "6862 samples in experience buffer. Filling...\n",
            "6946 samples in experience buffer. Filling...\n",
            "7030 samples in experience buffer. Filling...\n",
            "7109 samples in experience buffer. Filling...\n",
            "7191 samples in experience buffer. Filling...\n",
            "7289 samples in experience buffer. Filling...\n",
            "7367 samples in experience buffer. Filling...\n",
            "7455 samples in experience buffer. Filling...\n",
            "7524 samples in experience buffer. Filling...\n",
            "7617 samples in experience buffer. Filling...\n",
            "7698 samples in experience buffer. Filling...\n",
            "7801 samples in experience buffer. Filling...\n",
            "7907 samples in experience buffer. Filling...\n",
            "7990 samples in experience buffer. Filling...\n",
            "8090 samples in experience buffer. Filling...\n",
            "8183 samples in experience buffer. Filling...\n",
            "8285 samples in experience buffer. Filling...\n",
            "8353 samples in experience buffer. Filling...\n",
            "8499 samples in experience buffer. Filling...\n",
            "8619 samples in experience buffer. Filling...\n",
            "8747 samples in experience buffer. Filling...\n",
            "8808 samples in experience buffer. Filling...\n",
            "8885 samples in experience buffer. Filling...\n",
            "8965 samples in experience buffer. Filling...\n",
            "9054 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "9126 samples in experience buffer. Filling...\n",
            "9214 samples in experience buffer. Filling...\n",
            "9288 samples in experience buffer. Filling...\n",
            "9370 samples in experience buffer. Filling...\n",
            "9432 samples in experience buffer. Filling...\n",
            "9516 samples in experience buffer. Filling...\n",
            "9633 samples in experience buffer. Filling...\n",
            "9690 samples in experience buffer. Filling...\n",
            "9756 samples in experience buffer. Filling...\n",
            "9856 samples in experience buffer. Filling...\n",
            "9957 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n",
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 1.5 K  | train\n",
            "1 | target_q_net | DQN  | 1.5 K  | train\n",
            "----------------------------------------------\n",
            "3.0 K     Trainable params\n",
            "0         Non-trainable params\n",
            "3.0 K     Total params\n",
            "0.012     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "531a6ea2b49d4fe180e195bc750346cc",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 08:55:03,853] Trial 13 finished with value: -32.616188049316406 and parameters: {'lr': 0.014942508430231415, 'gamma': 0.7117141013262079, 'hidden_size': 32, 'eps_end': 0.1450575079501096}. Best is trial 9 with value: 252.13320922851562.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "77 samples in experience buffer. Filling...\n",
            "147 samples in experience buffer. Filling...\n",
            "213 samples in experience buffer. Filling...\n",
            "307 samples in experience buffer. Filling...\n",
            "423 samples in experience buffer. Filling...\n",
            "541 samples in experience buffer. Filling...\n",
            "644 samples in experience buffer. Filling...\n",
            "730 samples in experience buffer. Filling...\n",
            "796 samples in experience buffer. Filling...\n",
            "896 samples in experience buffer. Filling...\n",
            "962 samples in experience buffer. Filling...\n",
            "1039 samples in experience buffer. Filling...\n",
            "1112 samples in experience buffer. Filling...\n",
            "1209 samples in experience buffer. Filling...\n",
            "1279 samples in experience buffer. Filling...\n",
            "1356 samples in experience buffer. Filling...\n",
            "1430 samples in experience buffer. Filling...\n",
            "1552 samples in experience buffer. Filling...\n",
            "1658 samples in experience buffer. Filling...\n",
            "1781 samples in experience buffer. Filling...\n",
            "1857 samples in experience buffer. Filling...\n",
            "1953 samples in experience buffer. Filling...\n",
            "2045 samples in experience buffer. Filling...\n",
            "2113 samples in experience buffer. Filling...\n",
            "2217 samples in experience buffer. Filling...\n",
            "2287 samples in experience buffer. Filling...\n",
            "2360 samples in experience buffer. Filling...\n",
            "2454 samples in experience buffer. Filling...\n",
            "2545 samples in experience buffer. Filling...\n",
            "2611 samples in experience buffer. Filling...\n",
            "2733 samples in experience buffer. Filling...\n",
            "2801 samples in experience buffer. Filling...\n",
            "2920 samples in experience buffer. Filling...\n",
            "2998 samples in experience buffer. Filling...\n",
            "3108 samples in experience buffer. Filling...\n",
            "3217 samples in experience buffer. Filling...\n",
            "3324 samples in experience buffer. Filling...\n",
            "3432 samples in experience buffer. Filling...\n",
            "3504 samples in experience buffer. Filling...\n",
            "3600 samples in experience buffer. Filling...\n",
            "3663 samples in experience buffer. Filling...\n",
            "3763 samples in experience buffer. Filling...\n",
            "3871 samples in experience buffer. Filling...\n",
            "3935 samples in experience buffer. Filling...\n",
            "4043 samples in experience buffer. Filling...\n",
            "4110 samples in experience buffer. Filling...\n",
            "4213 samples in experience buffer. Filling...\n",
            "4311 samples in experience buffer. Filling...\n",
            "4403 samples in experience buffer. Filling...\n",
            "4482 samples in experience buffer. Filling...\n",
            "4557 samples in experience buffer. Filling...\n",
            "4637 samples in experience buffer. Filling...\n",
            "4752 samples in experience buffer. Filling...\n",
            "4871 samples in experience buffer. Filling...\n",
            "4935 samples in experience buffer. Filling...\n",
            "5043 samples in experience buffer. Filling...\n",
            "5115 samples in experience buffer. Filling...\n",
            "5225 samples in experience buffer. Filling...\n",
            "5329 samples in experience buffer. Filling...\n",
            "5442 samples in experience buffer. Filling...\n",
            "5506 samples in experience buffer. Filling...\n",
            "5631 samples in experience buffer. Filling...\n",
            "5725 samples in experience buffer. Filling...\n",
            "5810 samples in experience buffer. Filling...\n",
            "5903 samples in experience buffer. Filling...\n",
            "5993 samples in experience buffer. Filling...\n",
            "6088 samples in experience buffer. Filling...\n",
            "6168 samples in experience buffer. Filling...\n",
            "6271 samples in experience buffer. Filling...\n",
            "6364 samples in experience buffer. Filling...\n",
            "6464 samples in experience buffer. Filling...\n",
            "6552 samples in experience buffer. Filling...\n",
            "6641 samples in experience buffer. Filling...\n",
            "6740 samples in experience buffer. Filling...\n",
            "6848 samples in experience buffer. Filling...\n",
            "6929 samples in experience buffer. Filling...\n",
            "6982 samples in experience buffer. Filling...\n",
            "7060 samples in experience buffer. Filling...\n",
            "7161 samples in experience buffer. Filling...\n",
            "7277 samples in experience buffer. Filling...\n",
            "7337 samples in experience buffer. Filling...\n",
            "7455 samples in experience buffer. Filling...\n",
            "7517 samples in experience buffer. Filling...\n",
            "7594 samples in experience buffer. Filling...\n",
            "7693 samples in experience buffer. Filling...\n",
            "7807 samples in experience buffer. Filling...\n",
            "7932 samples in experience buffer. Filling...\n",
            "8026 samples in experience buffer. Filling...\n",
            "8140 samples in experience buffer. Filling...\n",
            "8267 samples in experience buffer. Filling...\n",
            "8391 samples in experience buffer. Filling...\n",
            "8496 samples in experience buffer. Filling...\n",
            "8596 samples in experience buffer. Filling...\n",
            "8684 samples in experience buffer. Filling...\n",
            "8814 samples in experience buffer. Filling...\n",
            "8890 samples in experience buffer. Filling...\n",
            "9027 samples in experience buffer. Filling...\n",
            "9135 samples in experience buffer. Filling...\n",
            "9256 samples in experience buffer. Filling...\n",
            "9339 samples in experience buffer. Filling...\n",
            "9457 samples in experience buffer. Filling...\n",
            "9529 samples in experience buffer. Filling...\n",
            "9664 samples in experience buffer. Filling...\n",
            "9730 samples in experience buffer. Filling...\n",
            "9845 samples in experience buffer. Filling...\n",
            "9967 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n",
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 18.2 K | train\n",
            "1 | target_q_net | DQN  | 18.2 K | train\n",
            "----------------------------------------------\n",
            "36.4 K    Trainable params\n",
            "0         Non-trainable params\n",
            "36.4 K    Total params\n",
            "0.145     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "221f5ab97348456ba167f669844f4564",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 08:57:52,327] Trial 14 finished with value: -50.66516876220703 and parameters: {'lr': 2.352909962855123e-05, 'gamma': 0.5367855145107857, 'hidden_size': 128, 'eps_end': 0.17069579341026087}. Best is trial 9 with value: 252.13320922851562.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "73 samples in experience buffer. Filling...\n",
            "179 samples in experience buffer. Filling...\n",
            "258 samples in experience buffer. Filling...\n",
            "387 samples in experience buffer. Filling...\n",
            "499 samples in experience buffer. Filling...\n",
            "569 samples in experience buffer. Filling...\n",
            "653 samples in experience buffer. Filling...\n",
            "721 samples in experience buffer. Filling...\n",
            "792 samples in experience buffer. Filling...\n",
            "873 samples in experience buffer. Filling...\n",
            "947 samples in experience buffer. Filling...\n",
            "1020 samples in experience buffer. Filling...\n",
            "1088 samples in experience buffer. Filling...\n",
            "1184 samples in experience buffer. Filling...\n",
            "1263 samples in experience buffer. Filling...\n",
            "1351 samples in experience buffer. Filling...\n",
            "1465 samples in experience buffer. Filling...\n",
            "1575 samples in experience buffer. Filling...\n",
            "1684 samples in experience buffer. Filling...\n",
            "1784 samples in experience buffer. Filling...\n",
            "1859 samples in experience buffer. Filling...\n",
            "1925 samples in experience buffer. Filling...\n",
            "2001 samples in experience buffer. Filling...\n",
            "2093 samples in experience buffer. Filling...\n",
            "2169 samples in experience buffer. Filling...\n",
            "2262 samples in experience buffer. Filling...\n",
            "2368 samples in experience buffer. Filling...\n",
            "2429 samples in experience buffer. Filling...\n",
            "2524 samples in experience buffer. Filling...\n",
            "2602 samples in experience buffer. Filling...\n",
            "2698 samples in experience buffer. Filling...\n",
            "2796 samples in experience buffer. Filling...\n",
            "2871 samples in experience buffer. Filling...\n",
            "2947 samples in experience buffer. Filling...\n",
            "3058 samples in experience buffer. Filling...\n",
            "3172 samples in experience buffer. Filling...\n",
            "3262 samples in experience buffer. Filling...\n",
            "3327 samples in experience buffer. Filling...\n",
            "3403 samples in experience buffer. Filling...\n",
            "3496 samples in experience buffer. Filling...\n",
            "3566 samples in experience buffer. Filling...\n",
            "3682 samples in experience buffer. Filling...\n",
            "3781 samples in experience buffer. Filling...\n",
            "3875 samples in experience buffer. Filling...\n",
            "3952 samples in experience buffer. Filling...\n",
            "4052 samples in experience buffer. Filling...\n",
            "4164 samples in experience buffer. Filling...\n",
            "4285 samples in experience buffer. Filling...\n",
            "4381 samples in experience buffer. Filling...\n",
            "4492 samples in experience buffer. Filling...\n",
            "4574 samples in experience buffer. Filling...\n",
            "4642 samples in experience buffer. Filling...\n",
            "4735 samples in experience buffer. Filling...\n",
            "4856 samples in experience buffer. Filling...\n",
            "4940 samples in experience buffer. Filling...\n",
            "5045 samples in experience buffer. Filling...\n",
            "5110 samples in experience buffer. Filling...\n",
            "5174 samples in experience buffer. Filling...\n",
            "5234 samples in experience buffer. Filling...\n",
            "5302 samples in experience buffer. Filling...\n",
            "5396 samples in experience buffer. Filling...\n",
            "5460 samples in experience buffer. Filling...\n",
            "5565 samples in experience buffer. Filling...\n",
            "5652 samples in experience buffer. Filling...\n",
            "5752 samples in experience buffer. Filling...\n",
            "5825 samples in experience buffer. Filling...\n",
            "5905 samples in experience buffer. Filling...\n",
            "6033 samples in experience buffer. Filling...\n",
            "6110 samples in experience buffer. Filling...\n",
            "6202 samples in experience buffer. Filling...\n",
            "6289 samples in experience buffer. Filling...\n",
            "6354 samples in experience buffer. Filling...\n",
            "6451 samples in experience buffer. Filling...\n",
            "6571 samples in experience buffer. Filling...\n",
            "6637 samples in experience buffer. Filling...\n",
            "6732 samples in experience buffer. Filling...\n",
            "6802 samples in experience buffer. Filling...\n",
            "6879 samples in experience buffer. Filling...\n",
            "6956 samples in experience buffer. Filling...\n",
            "7081 samples in experience buffer. Filling...\n",
            "7149 samples in experience buffer. Filling...\n",
            "7207 samples in experience buffer. Filling...\n",
            "7293 samples in experience buffer. Filling...\n",
            "7405 samples in experience buffer. Filling...\n",
            "7466 samples in experience buffer. Filling...\n",
            "7574 samples in experience buffer. Filling...\n",
            "7672 samples in experience buffer. Filling...\n",
            "7768 samples in experience buffer. Filling...\n",
            "7897 samples in experience buffer. Filling...\n",
            "7954 samples in experience buffer. Filling...\n",
            "8044 samples in experience buffer. Filling...\n",
            "8107 samples in experience buffer. Filling...\n",
            "8172 samples in experience buffer. Filling...\n",
            "8305 samples in experience buffer. Filling...\n",
            "8381 samples in experience buffer. Filling...\n",
            "8463 samples in experience buffer. Filling...\n",
            "8541 samples in experience buffer. Filling...\n",
            "8623 samples in experience buffer. Filling...\n",
            "8741 samples in experience buffer. Filling...\n",
            "8834 samples in experience buffer. Filling...\n",
            "8921 samples in experience buffer. Filling...\n",
            "8994 samples in experience buffer. Filling...\n",
            "9079 samples in experience buffer. Filling...\n",
            "9192 samples in experience buffer. Filling...\n",
            "9300 samples in experience buffer. Filling...\n",
            "9389 samples in experience buffer. Filling...\n",
            "9455 samples in experience buffer. Filling...\n",
            "9585 samples in experience buffer. Filling...\n",
            "9683 samples in experience buffer. Filling...\n",
            "9781 samples in experience buffer. Filling...\n",
            "9875 samples in experience buffer. Filling...\n",
            "9952 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n",
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 18.2 K | train\n",
            "1 | target_q_net | DQN  | 18.2 K | train\n",
            "----------------------------------------------\n",
            "36.4 K    Trainable params\n",
            "0         Non-trainable params\n",
            "36.4 K    Total params\n",
            "0.145     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "842fd7666f434bcdae9569621de486e6",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 09:03:26,970] Trial 15 finished with value: -90.5401611328125 and parameters: {'lr': 0.00032400638091776353, 'gamma': 0.9120063424733793, 'hidden_size': 128, 'eps_end': 0.13540534582189656}. Best is trial 9 with value: 252.13320922851562.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "122 samples in experience buffer. Filling...\n",
            "207 samples in experience buffer. Filling...\n",
            "296 samples in experience buffer. Filling...\n",
            "372 samples in experience buffer. Filling...\n",
            "458 samples in experience buffer. Filling...\n",
            "577 samples in experience buffer. Filling...\n",
            "669 samples in experience buffer. Filling...\n",
            "800 samples in experience buffer. Filling...\n",
            "905 samples in experience buffer. Filling...\n",
            "990 samples in experience buffer. Filling...\n",
            "1088 samples in experience buffer. Filling...\n",
            "1200 samples in experience buffer. Filling...\n",
            "1285 samples in experience buffer. Filling...\n",
            "1390 samples in experience buffer. Filling...\n",
            "1459 samples in experience buffer. Filling...\n",
            "1544 samples in experience buffer. Filling...\n",
            "1614 samples in experience buffer. Filling...\n",
            "1682 samples in experience buffer. Filling...\n",
            "1772 samples in experience buffer. Filling...\n",
            "1886 samples in experience buffer. Filling...\n",
            "1995 samples in experience buffer. Filling...\n",
            "2062 samples in experience buffer. Filling...\n",
            "2183 samples in experience buffer. Filling...\n",
            "2300 samples in experience buffer. Filling...\n",
            "2359 samples in experience buffer. Filling...\n",
            "2445 samples in experience buffer. Filling...\n",
            "2523 samples in experience buffer. Filling...\n",
            "2583 samples in experience buffer. Filling...\n",
            "2687 samples in experience buffer. Filling...\n",
            "2772 samples in experience buffer. Filling...\n",
            "2843 samples in experience buffer. Filling...\n",
            "2936 samples in experience buffer. Filling...\n",
            "3071 samples in experience buffer. Filling...\n",
            "3177 samples in experience buffer. Filling...\n",
            "3247 samples in experience buffer. Filling...\n",
            "3327 samples in experience buffer. Filling...\n",
            "3397 samples in experience buffer. Filling...\n",
            "3503 samples in experience buffer. Filling...\n",
            "3572 samples in experience buffer. Filling...\n",
            "3653 samples in experience buffer. Filling...\n",
            "3756 samples in experience buffer. Filling...\n",
            "3854 samples in experience buffer. Filling...\n",
            "3922 samples in experience buffer. Filling...\n",
            "4020 samples in experience buffer. Filling...\n",
            "4099 samples in experience buffer. Filling...\n",
            "4198 samples in experience buffer. Filling...\n",
            "4330 samples in experience buffer. Filling...\n",
            "4398 samples in experience buffer. Filling...\n",
            "4490 samples in experience buffer. Filling...\n",
            "4569 samples in experience buffer. Filling...\n",
            "4632 samples in experience buffer. Filling...\n",
            "4757 samples in experience buffer. Filling...\n",
            "4852 samples in experience buffer. Filling...\n",
            "4965 samples in experience buffer. Filling...\n",
            "5051 samples in experience buffer. Filling...\n",
            "5169 samples in experience buffer. Filling...\n",
            "5305 samples in experience buffer. Filling...\n",
            "5408 samples in experience buffer. Filling...\n",
            "5494 samples in experience buffer. Filling...\n",
            "5564 samples in experience buffer. Filling...\n",
            "5640 samples in experience buffer. Filling...\n",
            "5713 samples in experience buffer. Filling...\n",
            "5817 samples in experience buffer. Filling...\n",
            "5959 samples in experience buffer. Filling...\n",
            "6088 samples in experience buffer. Filling...\n",
            "6194 samples in experience buffer. Filling...\n",
            "6262 samples in experience buffer. Filling...\n",
            "6376 samples in experience buffer. Filling...\n",
            "6462 samples in experience buffer. Filling...\n",
            "6537 samples in experience buffer. Filling...\n",
            "6606 samples in experience buffer. Filling...\n",
            "6699 samples in experience buffer. Filling...\n",
            "6784 samples in experience buffer. Filling...\n",
            "6871 samples in experience buffer. Filling...\n",
            "6989 samples in experience buffer. Filling...\n",
            "7055 samples in experience buffer. Filling...\n",
            "7138 samples in experience buffer. Filling...\n",
            "7214 samples in experience buffer. Filling...\n",
            "7297 samples in experience buffer. Filling...\n",
            "7361 samples in experience buffer. Filling...\n",
            "7466 samples in experience buffer. Filling...\n",
            "7551 samples in experience buffer. Filling...\n",
            "7631 samples in experience buffer. Filling...\n",
            "7759 samples in experience buffer. Filling...\n",
            "7879 samples in experience buffer. Filling...\n",
            "7963 samples in experience buffer. Filling...\n",
            "8038 samples in experience buffer. Filling...\n",
            "8135 samples in experience buffer. Filling...\n",
            "8217 samples in experience buffer. Filling...\n",
            "8284 samples in experience buffer. Filling...\n",
            "8352 samples in experience buffer. Filling...\n",
            "8437 samples in experience buffer. Filling...\n",
            "8543 samples in experience buffer. Filling...\n",
            "8620 samples in experience buffer. Filling...\n",
            "8727 samples in experience buffer. Filling...\n",
            "8842 samples in experience buffer. Filling...\n",
            "8920 samples in experience buffer. Filling...\n",
            "8978 samples in experience buffer. Filling...\n",
            "9061 samples in experience buffer. Filling...\n",
            "9149 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "9245 samples in experience buffer. Filling...\n",
            "9328 samples in experience buffer. Filling...\n",
            "9450 samples in experience buffer. Filling...\n",
            "9566 samples in experience buffer. Filling...\n",
            "9664 samples in experience buffer. Filling...\n",
            "9745 samples in experience buffer. Filling...\n",
            "9807 samples in experience buffer. Filling...\n",
            "9900 samples in experience buffer. Filling...\n",
            "9973 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 18.2 K | train\n",
            "1 | target_q_net | DQN  | 18.2 K | train\n",
            "----------------------------------------------\n",
            "36.4 K    Trainable params\n",
            "0         Non-trainable params\n",
            "36.4 K    Total params\n",
            "0.145     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a1176ba9f1f14d879d04f99877669a18",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 09:06:35,807] Trial 16 finished with value: -117.21269226074219 and parameters: {'lr': 0.007435679925556904, 'gamma': 0.4613738088805487, 'hidden_size': 128, 'eps_end': 0.17387336072535675}. Best is trial 9 with value: 252.13320922851562.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "109 samples in experience buffer. Filling...\n",
            "191 samples in experience buffer. Filling...\n",
            "293 samples in experience buffer. Filling...\n",
            "393 samples in experience buffer. Filling...\n",
            "472 samples in experience buffer. Filling...\n",
            "577 samples in experience buffer. Filling...\n",
            "665 samples in experience buffer. Filling...\n",
            "773 samples in experience buffer. Filling...\n",
            "867 samples in experience buffer. Filling...\n",
            "939 samples in experience buffer. Filling...\n",
            "1072 samples in experience buffer. Filling...\n",
            "1186 samples in experience buffer. Filling...\n",
            "1301 samples in experience buffer. Filling...\n",
            "2301 samples in experience buffer. Filling...\n",
            "2408 samples in experience buffer. Filling...\n",
            "2496 samples in experience buffer. Filling...\n",
            "2619 samples in experience buffer. Filling...\n",
            "2735 samples in experience buffer. Filling...\n",
            "2798 samples in experience buffer. Filling...\n",
            "2914 samples in experience buffer. Filling...\n",
            "3024 samples in experience buffer. Filling...\n",
            "3115 samples in experience buffer. Filling...\n",
            "3267 samples in experience buffer. Filling...\n",
            "3361 samples in experience buffer. Filling...\n",
            "3484 samples in experience buffer. Filling...\n",
            "3576 samples in experience buffer. Filling...\n",
            "3665 samples in experience buffer. Filling...\n",
            "3736 samples in experience buffer. Filling...\n",
            "3798 samples in experience buffer. Filling...\n",
            "3895 samples in experience buffer. Filling...\n",
            "3960 samples in experience buffer. Filling...\n",
            "4028 samples in experience buffer. Filling...\n",
            "4109 samples in experience buffer. Filling...\n",
            "4205 samples in experience buffer. Filling...\n",
            "4285 samples in experience buffer. Filling...\n",
            "4401 samples in experience buffer. Filling...\n",
            "4503 samples in experience buffer. Filling...\n",
            "4577 samples in experience buffer. Filling...\n",
            "4668 samples in experience buffer. Filling...\n",
            "4792 samples in experience buffer. Filling...\n",
            "4877 samples in experience buffer. Filling...\n",
            "4939 samples in experience buffer. Filling...\n",
            "5059 samples in experience buffer. Filling...\n",
            "5126 samples in experience buffer. Filling...\n",
            "5229 samples in experience buffer. Filling...\n",
            "5358 samples in experience buffer. Filling...\n",
            "5438 samples in experience buffer. Filling...\n",
            "5513 samples in experience buffer. Filling...\n",
            "5618 samples in experience buffer. Filling...\n",
            "5704 samples in experience buffer. Filling...\n",
            "5824 samples in experience buffer. Filling...\n",
            "5952 samples in experience buffer. Filling...\n",
            "6064 samples in experience buffer. Filling...\n",
            "6164 samples in experience buffer. Filling...\n",
            "6234 samples in experience buffer. Filling...\n",
            "6305 samples in experience buffer. Filling...\n",
            "6408 samples in experience buffer. Filling...\n",
            "6493 samples in experience buffer. Filling...\n",
            "6556 samples in experience buffer. Filling...\n",
            "6645 samples in experience buffer. Filling...\n",
            "6742 samples in experience buffer. Filling...\n",
            "6838 samples in experience buffer. Filling...\n",
            "6920 samples in experience buffer. Filling...\n",
            "6993 samples in experience buffer. Filling...\n",
            "7087 samples in experience buffer. Filling...\n",
            "7145 samples in experience buffer. Filling...\n",
            "7276 samples in experience buffer. Filling...\n",
            "7360 samples in experience buffer. Filling...\n",
            "7474 samples in experience buffer. Filling...\n",
            "7561 samples in experience buffer. Filling...\n",
            "7646 samples in experience buffer. Filling...\n",
            "7740 samples in experience buffer. Filling...\n",
            "7913 samples in experience buffer. Filling...\n",
            "7993 samples in experience buffer. Filling...\n",
            "8104 samples in experience buffer. Filling...\n",
            "8208 samples in experience buffer. Filling...\n",
            "8295 samples in experience buffer. Filling...\n",
            "8392 samples in experience buffer. Filling...\n",
            "8470 samples in experience buffer. Filling...\n",
            "8550 samples in experience buffer. Filling...\n",
            "8658 samples in experience buffer. Filling...\n",
            "8768 samples in experience buffer. Filling...\n",
            "8874 samples in experience buffer. Filling...\n",
            "8957 samples in experience buffer. Filling...\n",
            "9022 samples in experience buffer. Filling...\n",
            "9109 samples in experience buffer. Filling...\n",
            "9239 samples in experience buffer. Filling...\n",
            "9300 samples in experience buffer. Filling...\n",
            "9359 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "9420 samples in experience buffer. Filling...\n",
            "9524 samples in experience buffer. Filling...\n",
            "9609 samples in experience buffer. Filling...\n",
            "9669 samples in experience buffer. Filling...\n",
            "9769 samples in experience buffer. Filling...\n",
            "9876 samples in experience buffer. Filling...\n",
            "9967 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 1.5 K  | train\n",
            "1 | target_q_net | DQN  | 1.5 K  | train\n",
            "----------------------------------------------\n",
            "3.0 K     Trainable params\n",
            "0         Non-trainable params\n",
            "3.0 K     Total params\n",
            "0.012     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "682d1e307ad44395adf8e63f0a0ae7fc",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 10:47:25,335] Trial 17 finished with value: -219.07022094726562 and parameters: {'lr': 4.3877528163358176e-05, 'gamma': 0.7944064008909305, 'hidden_size': 32, 'eps_end': 0.09110725805950888}. Best is trial 9 with value: 252.13320922851562.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "86 samples in experience buffer. Filling...\n",
            "163 samples in experience buffer. Filling...\n",
            "302 samples in experience buffer. Filling...\n",
            "397 samples in experience buffer. Filling...\n",
            "508 samples in experience buffer. Filling...\n",
            "582 samples in experience buffer. Filling...\n",
            "644 samples in experience buffer. Filling...\n",
            "738 samples in experience buffer. Filling...\n",
            "857 samples in experience buffer. Filling...\n",
            "1003 samples in experience buffer. Filling...\n",
            "1093 samples in experience buffer. Filling...\n",
            "1157 samples in experience buffer. Filling...\n",
            "1227 samples in experience buffer. Filling...\n",
            "1297 samples in experience buffer. Filling...\n",
            "1385 samples in experience buffer. Filling...\n",
            "1455 samples in experience buffer. Filling...\n",
            "1559 samples in experience buffer. Filling...\n",
            "1641 samples in experience buffer. Filling...\n",
            "1732 samples in experience buffer. Filling...\n",
            "1824 samples in experience buffer. Filling...\n",
            "1916 samples in experience buffer. Filling...\n",
            "1985 samples in experience buffer. Filling...\n",
            "2071 samples in experience buffer. Filling...\n",
            "2161 samples in experience buffer. Filling...\n",
            "2242 samples in experience buffer. Filling...\n",
            "2370 samples in experience buffer. Filling...\n",
            "2445 samples in experience buffer. Filling...\n",
            "2531 samples in experience buffer. Filling...\n",
            "2650 samples in experience buffer. Filling...\n",
            "2732 samples in experience buffer. Filling...\n",
            "2835 samples in experience buffer. Filling...\n",
            "2924 samples in experience buffer. Filling...\n",
            "3002 samples in experience buffer. Filling...\n",
            "3069 samples in experience buffer. Filling...\n",
            "3131 samples in experience buffer. Filling...\n",
            "3241 samples in experience buffer. Filling...\n",
            "3330 samples in experience buffer. Filling...\n",
            "3424 samples in experience buffer. Filling...\n",
            "3489 samples in experience buffer. Filling...\n",
            "3557 samples in experience buffer. Filling...\n",
            "3686 samples in experience buffer. Filling...\n",
            "3744 samples in experience buffer. Filling...\n",
            "3860 samples in experience buffer. Filling...\n",
            "3927 samples in experience buffer. Filling...\n",
            "3995 samples in experience buffer. Filling...\n",
            "4066 samples in experience buffer. Filling...\n",
            "4163 samples in experience buffer. Filling...\n",
            "4232 samples in experience buffer. Filling...\n",
            "4302 samples in experience buffer. Filling...\n",
            "4384 samples in experience buffer. Filling...\n",
            "4501 samples in experience buffer. Filling...\n",
            "4585 samples in experience buffer. Filling...\n",
            "4671 samples in experience buffer. Filling...\n",
            "4767 samples in experience buffer. Filling...\n",
            "4865 samples in experience buffer. Filling...\n",
            "4966 samples in experience buffer. Filling...\n",
            "5096 samples in experience buffer. Filling...\n",
            "5193 samples in experience buffer. Filling...\n",
            "5286 samples in experience buffer. Filling...\n",
            "5387 samples in experience buffer. Filling...\n",
            "5482 samples in experience buffer. Filling...\n",
            "5549 samples in experience buffer. Filling...\n",
            "5633 samples in experience buffer. Filling...\n",
            "5756 samples in experience buffer. Filling...\n",
            "5837 samples in experience buffer. Filling...\n",
            "5991 samples in experience buffer. Filling...\n",
            "6075 samples in experience buffer. Filling...\n",
            "6196 samples in experience buffer. Filling...\n",
            "6335 samples in experience buffer. Filling...\n",
            "6395 samples in experience buffer. Filling...\n",
            "6463 samples in experience buffer. Filling...\n",
            "6522 samples in experience buffer. Filling...\n",
            "6591 samples in experience buffer. Filling...\n",
            "6686 samples in experience buffer. Filling...\n",
            "6795 samples in experience buffer. Filling...\n",
            "6879 samples in experience buffer. Filling...\n",
            "7009 samples in experience buffer. Filling...\n",
            "7093 samples in experience buffer. Filling...\n",
            "7171 samples in experience buffer. Filling...\n",
            "7258 samples in experience buffer. Filling...\n",
            "7350 samples in experience buffer. Filling...\n",
            "7435 samples in experience buffer. Filling...\n",
            "7534 samples in experience buffer. Filling...\n",
            "7617 samples in experience buffer. Filling...\n",
            "7701 samples in experience buffer. Filling...\n",
            "7840 samples in experience buffer. Filling...\n",
            "7971 samples in experience buffer. Filling...\n",
            "8072 samples in experience buffer. Filling...\n",
            "8160 samples in experience buffer. Filling...\n",
            "8219 samples in experience buffer. Filling...\n",
            "8333 samples in experience buffer. Filling...\n",
            "8413 samples in experience buffer. Filling...\n",
            "8546 samples in experience buffer. Filling...\n",
            "8647 samples in experience buffer. Filling...\n",
            "8713 samples in experience buffer. Filling...\n",
            "8817 samples in experience buffer. Filling...\n",
            "8910 samples in experience buffer. Filling...\n",
            "8983 samples in experience buffer. Filling...\n",
            "9084 samples in experience buffer. Filling...\n",
            "9157 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "9274 samples in experience buffer. Filling...\n",
            "9357 samples in experience buffer. Filling...\n",
            "9425 samples in experience buffer. Filling...\n",
            "9487 samples in experience buffer. Filling...\n",
            "9547 samples in experience buffer. Filling...\n",
            "9647 samples in experience buffer. Filling...\n",
            "9734 samples in experience buffer. Filling...\n",
            "9833 samples in experience buffer. Filling...\n",
            "9955 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n",
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 5.0 K  | train\n",
            "1 | target_q_net | DQN  | 5.0 K  | train\n",
            "----------------------------------------------\n",
            "10.0 K    Trainable params\n",
            "0         Non-trainable params\n",
            "10.0 K    Total params\n",
            "0.040     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c4439e1696914703a61166e93a5f4c1a",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 10:51:55,397] Trial 18 finished with value: -200.4228515625 and parameters: {'lr': 0.0004472710024737123, 'gamma': 0.6611642974344647, 'hidden_size': 64, 'eps_end': 0.002903793341214017}. Best is trial 9 with value: 252.13320922851562.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 samples in experience buffer. Filling...\n",
            "57 samples in experience buffer. Filling...\n",
            "177 samples in experience buffer. Filling...\n",
            "306 samples in experience buffer. Filling...\n",
            "435 samples in experience buffer. Filling...\n",
            "521 samples in experience buffer. Filling...\n",
            "600 samples in experience buffer. Filling...\n",
            "669 samples in experience buffer. Filling...\n",
            "773 samples in experience buffer. Filling...\n",
            "878 samples in experience buffer. Filling...\n",
            "1008 samples in experience buffer. Filling...\n",
            "1083 samples in experience buffer. Filling...\n",
            "1155 samples in experience buffer. Filling...\n",
            "1294 samples in experience buffer. Filling...\n",
            "1384 samples in experience buffer. Filling...\n",
            "1469 samples in experience buffer. Filling...\n",
            "1557 samples in experience buffer. Filling...\n",
            "1691 samples in experience buffer. Filling...\n",
            "1753 samples in experience buffer. Filling...\n",
            "1862 samples in experience buffer. Filling...\n",
            "1954 samples in experience buffer. Filling...\n",
            "2035 samples in experience buffer. Filling...\n",
            "2166 samples in experience buffer. Filling...\n",
            "2231 samples in experience buffer. Filling...\n",
            "2337 samples in experience buffer. Filling...\n",
            "2424 samples in experience buffer. Filling...\n",
            "2552 samples in experience buffer. Filling...\n",
            "2655 samples in experience buffer. Filling...\n",
            "2759 samples in experience buffer. Filling...\n",
            "2851 samples in experience buffer. Filling...\n",
            "2939 samples in experience buffer. Filling...\n",
            "3035 samples in experience buffer. Filling...\n",
            "3141 samples in experience buffer. Filling...\n",
            "3253 samples in experience buffer. Filling...\n",
            "3316 samples in experience buffer. Filling...\n",
            "3423 samples in experience buffer. Filling...\n",
            "3497 samples in experience buffer. Filling...\n",
            "3599 samples in experience buffer. Filling...\n",
            "3663 samples in experience buffer. Filling...\n",
            "3769 samples in experience buffer. Filling...\n",
            "3827 samples in experience buffer. Filling...\n",
            "3914 samples in experience buffer. Filling...\n",
            "4033 samples in experience buffer. Filling...\n",
            "4126 samples in experience buffer. Filling...\n",
            "4229 samples in experience buffer. Filling...\n",
            "4321 samples in experience buffer. Filling...\n",
            "4411 samples in experience buffer. Filling...\n",
            "4491 samples in experience buffer. Filling...\n",
            "4622 samples in experience buffer. Filling...\n",
            "4733 samples in experience buffer. Filling...\n",
            "4811 samples in experience buffer. Filling...\n",
            "4908 samples in experience buffer. Filling...\n",
            "4969 samples in experience buffer. Filling...\n",
            "5037 samples in experience buffer. Filling...\n",
            "5135 samples in experience buffer. Filling...\n",
            "5235 samples in experience buffer. Filling...\n",
            "5343 samples in experience buffer. Filling...\n",
            "5458 samples in experience buffer. Filling...\n",
            "5568 samples in experience buffer. Filling...\n",
            "5663 samples in experience buffer. Filling...\n",
            "5784 samples in experience buffer. Filling...\n",
            "5852 samples in experience buffer. Filling...\n",
            "5930 samples in experience buffer. Filling...\n",
            "6011 samples in experience buffer. Filling...\n",
            "6067 samples in experience buffer. Filling...\n",
            "6140 samples in experience buffer. Filling...\n",
            "6232 samples in experience buffer. Filling...\n",
            "6360 samples in experience buffer. Filling...\n",
            "6468 samples in experience buffer. Filling...\n",
            "6586 samples in experience buffer. Filling...\n",
            "6683 samples in experience buffer. Filling...\n",
            "6794 samples in experience buffer. Filling...\n",
            "6895 samples in experience buffer. Filling...\n",
            "6973 samples in experience buffer. Filling...\n",
            "7072 samples in experience buffer. Filling...\n",
            "7160 samples in experience buffer. Filling...\n",
            "7257 samples in experience buffer. Filling...\n",
            "7382 samples in experience buffer. Filling...\n",
            "7484 samples in experience buffer. Filling...\n",
            "7605 samples in experience buffer. Filling...\n",
            "7681 samples in experience buffer. Filling...\n",
            "7811 samples in experience buffer. Filling...\n",
            "7886 samples in experience buffer. Filling...\n",
            "7977 samples in experience buffer. Filling...\n",
            "8050 samples in experience buffer. Filling...\n",
            "8142 samples in experience buffer. Filling...\n",
            "8238 samples in experience buffer. Filling...\n",
            "8356 samples in experience buffer. Filling...\n",
            "8463 samples in experience buffer. Filling...\n",
            "8540 samples in experience buffer. Filling...\n",
            "8616 samples in experience buffer. Filling...\n",
            "8704 samples in experience buffer. Filling...\n",
            "8768 samples in experience buffer. Filling...\n",
            "8833 samples in experience buffer. Filling...\n",
            "8937 samples in experience buffer. Filling...\n",
            "9005 samples in experience buffer. Filling...\n",
            "9092 samples in experience buffer. Filling...\n",
            "9185 samples in experience buffer. Filling...\n",
            "9287 samples in experience buffer. Filling...\n",
            "9406 samples in experience buffer. Filling...\n",
            "9473 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "GPU available: True (cuda), used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "HPU available: False, using: 0 HPUs\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "9579 samples in experience buffer. Filling...\n",
            "9653 samples in experience buffer. Filling...\n",
            "9716 samples in experience buffer. Filling...\n",
            "9784 samples in experience buffer. Filling...\n",
            "9867 samples in experience buffer. Filling...\n",
            "9968 samples in experience buffer. Filling...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "\n",
            "  | Name         | Type | Params | Mode \n",
            "----------------------------------------------\n",
            "0 | q_net        | DQN  | 18.2 K | train\n",
            "1 | target_q_net | DQN  | 18.2 K | train\n",
            "----------------------------------------------\n",
            "36.4 K    Trainable params\n",
            "0         Non-trainable params\n",
            "36.4 K    Total params\n",
            "0.145     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "58da8728f6924d75a2d6b6fe3b2d2a8f",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: |          | 0/? [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`Trainer.fit` stopped: `max_epochs=200` reached.\n",
            "[I 2024-07-02 10:55:03,267] Trial 19 finished with value: -183.309814453125 and parameters: {'lr': 0.09511721044852364, 'gamma': 0.44616155190275203, 'hidden_size': 128, 'eps_end': 0.19012410024841195}. Best is trial 9 with value: 252.13320922851562.\n"
          ]
        }
      ],
      "source": [
        "study.optimize(objective, n_trials=20)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oG0rQKyT5H-P"
      },
      "source": [
        "#### Select and use the best hyperparameters"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "metadata": {
        "id": "ZiCp91CbPmrn"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'lr': 0.0036502063949420765,\n",
              " 'gamma': 0.6207765698639972,\n",
              " 'hidden_size': 128,\n",
              " 'eps_end': 0.08699195823968492}"
            ]
          },
          "execution_count": 48,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "ename": "",
          "evalue": "",
          "output_type": "error",
          "traceback": [
            "\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n",
            "\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n",
            "\u001b[1;31mClick <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. \n",
            "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
          ]
        }
      ],
      "source": [
        "study.best_params"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VyXUmpt05WaK"
      },
      "outputs": [],
      "source": [
        "# Start tensorboard.\n",
        "!rm -r /content/lightning_logs/\n",
        "!rm -r /content/videos/\n",
        "%load_ext tensorboard\n",
        "%tensorboard --logdir /content/lightning_logs/"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fq8UhUU0QUP-"
      },
      "outputs": [],
      "source": [
        "algo = DeepQLearning('LunarLander-v2', **study.best_params)\n",
        "\n",
        "trainer = Trainer(\n",
        "  gpus=num_gpus, \n",
        "  max_epochs=1000,\n",
        "  track_grad_norm=2,\n",
        ")\n",
        "\n",
        "trainer.fit(algo)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jD3x39w71xWR"
      },
      "source": [
        "#### Check the resulting policy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TSElgv7I4fE6"
      },
      "outputs": [],
      "source": [
        "display_video(episode=500)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.4"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}