KNN_Random_Ref_Randomized_Attempt.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"kernelspec":{"display_name":"Python 3.8 (tensorflow)","language":"python","name":"machinelearningintroduction"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.8.5"},"colab":{"name":"KNN_Random_Ref_Randomized_Attempt.ipynb","provenance":[],"collapsed_sections":[]}},"cells":[{"cell_type":"code","metadata":{"id":"7QxcSzi_P6L5"},"source":["import pandas as pd\n","import numpy as np\n","import time\n","import matplotlib.pyplot as plt\n","\n","from sklearn.neighbors import NearestNeighbors"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"u3rcPJFzP6L8"},"source":["# Load dataset\n","dataset = pd.read_csv('final_dataset.csv')\n","df = dataset.copy()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"9ZZDPtGaP6L9"},"source":["# Preprocessing\n","df.drop(columns=['Unnamed: 0'], inplace=True)\n","df = df.astype(df.values).astype('float32')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"zIyV-kFlP6L9","executionInfo":{"status":"ok","timestamp":1623854171894,"user_tz":-120,"elapsed":619,"user":{"displayName":"Uczenie Maszynowe","photoUrl":"","userId":"12091857420517364173"}}},"source":["# Extracting a reference dataset with 250 000 samples\n","df_ref = df.sample(n=250_000)\n","df_ref.shape[0]"],"execution_count":2,"outputs":[]},{"cell_type":"code","metadata":{"id":"Byygi9K3P6L-"},"source":["# Saving reference dataset's indexes to a file\n","reference_dataset_indexes = list()\n","\n","\n","for index, _ in df_ref.iterrows():\n","    reference_dataset_indexes.append(index)\n","\n","import csv\n","  \n","with open('[KNN]reference_indexes.csv', 'w', newline='') as f:\n","    write = csv.writer(f) \n","    write.writerow(['index'])\n","    write.writerows([element] for element in sorted(reference_dataset_indexes))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"oJ43W6CyP6L-"},"source":["# Exlude reference dataset's samples from entire data set to obtain data set for testing purposes\n","df_testing = df[~df.isin(df_ref)].dropna()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"GIXSJprRP6L_","outputId":"6c52a423-6dd2-4b72-c1fc-afbb0035a5d7"},"source":["print(f\"Entire Data Set has {df.shape[0]} samples.\")\n","print(f\"Reference Data Set has {df_ref.shape[0]} samples.\")\n","print(f\"Testing Data Set has {df_testing.shape[0]} samples.\")"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Entire Data Set has 3577296 samples.\n","Reference Data Set has 250000 samples.\n","Testing Data Set has 3327296 samples.\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"eBxuTMqoP6L_"},"source":["# Compute distances \n","K = 5\n","neighbors = NearestNeighbors(n_neighbors=K)\n","neighbors.fit(df_ref.values)\n","distances, _ = neighbors.kneighbors(df_ref.values)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"l9lLQnh0P6MA"},"source":["distances_nei = distances.copy()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"gbGv3CXNP6MA","outputId":"196bdda4-1162-42f2-9c31-65bbfe558ffe"},"source":["plt.plot(distances_nei.mean(axis=1))\n","plt.plot([0,250_000], [np.percentile(distances_nei, 99.5), np.percentile(distances_nei, 99.5)])"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[<matplotlib.lines.Line2D at 0x20b95ead3d0>]"]},"metadata":{"tags":[]},"execution_count":20},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXgAAAD4CAYAAADmWv3KAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAj50lEQVR4nO3deXRc5X038O8Pu05PA0kgmJwU08i0EOKchgQESU9KQl/SYkNaaFPOMTQNNaQuLRSSvG1QoE7gBRr2LRgc24BZEmwWAzaSbWzjBe8eYS2WbFmyJFuyrF22Nmt/3j/mjj0a3Zm5d+auz3w/5/h4dOfO3OeZO/c3z31WUUqBiIj0c5rfCSAiIncwwBMRaYoBnohIUwzwRESaYoAnItLUZL8OfPbZZ6u8vDy/Dk9EFErFxcXtSqmpVvb1LcDn5eUhEon4dXgiolASkUNW92UVDRGRphjgiYg0xQBPRKQpBngiIk0xwBMRaYoBnohIUwzwRESaYoAn8pFSCm8XN2JgeNTvpJCGGOCJfLSxqg3/9VYpHl1d5XdSSENpA7yIvCQirSKyN8nzIiLPikiNiJSJyCXOJ5NIT90DwwCA9t5Bn1NCOrJSgl8CYGaK52cBuMD4NxfAC9kni4iIspU2wCulNgPoTLHLdQBeVVE7AHxORL7oVAKJiCgzTtTBnwugIe7vRmPbBCIyV0QiIhJpa2tz4NBERJSMEwFeTLaZruStlFqolMpXSuVPnWpptksiIsqQEwG+EcB5cX9PA9DkwPsSEVEWnAjwKwD8yOhN8y0Ax5VSRx14XyIiykLaBT9E5A0AVwI4W0QaAfwKwB8AgFJqAYAiANcAqAHQD2COW4klIiLr0gZ4pdSNaZ5XAG53LEVEROQIjmQlItIUAzwRkaYY4ImINMUAT0SkKQZ4IiJNMcATEWmKAZ6ISFMM8EREmmKAJyLSFAM8EZGmGOCJiDTFAE9EpCkGeCIiTTHAExFpigGeiEhTDPBERJpigCci0hQDPBGRphjgA25kdAz/+cYe7G/u9jspRBQyDPABV93ai5WlTfjJ0hK/k0JEIcMA75IXt9ShrPGY38kgohw22e8E6OqBDyoBAPUPX+tzSogoV7EET0SkKQZ4ooBav68Fc17e5XcyKMRYRUMUULe+EvE7CRRyLMETkatuXbIbeQWFficjJzHAE5Gr1u9v9TsJOYsBPiSU8jsFRBQ2DPABJ+J3CijsfrasBDcs2OZ3MsgHbGQl8pEXd2bL9xxx/yAUSJZK8CIyU0SqRKRGRApMnv+siKwUkVIRqRCROc4nlUhfvFMjN6QN8CIyCcB8ALMAzABwo4jMSNjtdgCVSqmLAVwJ4AkRmeJwWolyQm1bL1p7BvxOhtZauwfw4AeVGB3Tu3HLShXN5QBqlFK1ACAiSwFcB6Aybh8F4AwREQCnA+gEMOJwWolywv95YpPfSdBewfJyfLS/Fd+5cCq+c+FUv5PjGitVNOcCaIj7u9HYFu85AF8B0ASgHMBdSqmxxDcSkbkiEhGRSFtbW4ZJJjJX3ngceQWF2HvkuN9JoYAbHo2GJ73L79YCvFntYOLncjWAEgB/DODrAJ4Tkc9MeJFSC5VS+Uqp/KlTg/GruWDTQeQVFKJ3kDccYbd2XwsAYJ3xPznr/pUVWM/PNlSsBPhGAOfF/T0N0ZJ6vDkAlquoGgB1AC5yJonuen3HIQBAV9+QzykhCraXt9Zz+oSQsRLgdwO4QESmGw2nswGsSNjnMICrAEBEvgDgywBqnUxorlPa30wSkdPSNrIqpUZE5A4AawBMAvCSUqpCRG4znl8A4AEAS0SkHNEqnbuVUu0upjtniGkNGRFRepYGOimligAUJWxbEPe4CcDfOJs0IgIApRSEHeUpA5yqgIgoQ209g4HuT88AT0SUoXveLcfiLXXYXB3Mbt8M8BYs2VqHovKjfieDiAIm1p8+qH0gONmYBfet5ALaRBQ+LMETkeM6egfxi+XlGBwZ9TspOY0leAMX1CByzkNF+7D8kyPI/9KZficlp+V8CT6x99ngyOiperUA4Q9Q7grluQ9JmlUoP1zrcj7AJ/ry/6zG954Mzmx+7P5M5LxcGVeQ8wHe7Af8UEe/9wlJUN/eh/beQb+TQeSa3+88jHWV4Z68LOg3AKyDNwTtB/3Kxzdi8mmCoruu8DspRK64591yAJr0TgtY/IjJ+RJ8kI0EdHQcEYUDAzwRuYZFFH8xwBOR8wJaZZFrGOCJiDTFAE8UcKzmoEwxwIdEEC7yjt5BbKnmOi4Ufk4NcArCdZkKA7whqP1Zg1SVeeOiHfjhizsxxt49ZFHQR4o6NeApSNdpvJwP8EHr/x5kB1p6AfAzo/S41GQw5HyAJyJnPLn2AC5/aN24bcEuvzsvUt+JkQDNZaVtgP+wohm9gyN+J4MoZzy7vhqtPdHpNcJyl5dtFVL860sbjuEfF2zHE2sPZJssx2gZ4A+29WLua8X477dKk+7T3juI6b8oREPnCQ9TRkRB4PRkYyKCNuPH7UBzj6PvnQ0tA/yJoegiA4c7k08atquuc1zD6tvFDVi9t9ntpBHZFvSGSifsqO1AXkEh6tr7/E6KVjjZmOHZj2r8TgJRznpvzxEA0UA//exPZ/Qe18/fiku/dCbmfX+Gk0kLNS1L8OStnoFhLP+kMaPXFrxThryCQodTRLmopOEYXtxS53cyAoUl+JAI8m16wfJyFJYdxQXnnIE/n/ZZW69durvBpVQReSeobcoswQdcEHsjvF/SNO7v1u4BAMCJYZ8XWA7wj6AXRkbHsKr8aLAKAwFKSi5igNdU7+AIDnW402D1+o5DrryvU5waZLO/uRvFh7oceS8vzN9wEP/+u0+wpsL/VZICWC5xXRB/yxjgNXXToh347mMbXXnvIH6R3TDz6Y/xgxe2+Z0My5q7o11+O/uGfE7JKSpnvi2nBOmum3XwmiprPO53EsghYQyRQQpyXujqH0L3QPAGVloqwYvITBGpEpEaESlIss+VIlIiIhUissnZZJJVw6NjWL/P/1t0siZVCVfXIHmoow8XzVvl2fH2Hjnu+vQBdy0twbz39rp6jEykDfAiMgnAfACzAMwAcKOIzEjY53MAngfwd0qprwK4wfmkTtTeO4iuDG9HNb128Jv11bj1lYjfyQi113ccwtVPbfb0mGH/PlY0Wb9jfHfPEQwMezNfy/7mbnz/N1tcmz4gSO3ZZqxU0VwOoEYpVQsAIrIUwHUAKuP2uQnAcqXUYQBQSrU6nVAz+Q9GJza64dJpeOyGiyc8H/QP3w2pRu+SNf8TkJJYWL6/e48cx/d/s8WXY7+2vR77m3vw0N//uenzrd3R6QP2HsnNKksrVTTnAojvrNxobIt3IYAzRWSjiBSLyI+cSqAVbxVnNsgmlQ37PfmNIgq9o8cHkj7n5I/UusoWXPXExnHVLfPer8Dvdh7O+D1D8huaMSsB3uzuMfFzmQzgUgDXArgawDwRuXDCG4nMFZGIiETa2tpsJ9ZLc5bsRpXNSYNODI1i/oYaV+r7vPgiljQcQ3vvYMavD1T/a/KVG/PBFywvx8G2PnT2Z99LKOxVYlZZCfCNAM6L+3sagCaTfVYrpfqUUu0ANgOYUGeilFqolMpXSuVPnTo10zR7pndw2Nb+T687gMfWVGG5Ma+GM+x9FbMJsdfP34q/tXCrnZgivxZ3qGntQU2rMzP3jY4pPFRYieYUpdFsra1sQV5BYdJ2o9V7j2L+htycEyloZYOa1l7kFRSitOFYBq8Ozs+HlTr43QAuEJHpAI4AmI1onXu89wE8JyKTAUwB8E0ATzmZ0DDoG4p2kxr0cURnthdKqtvtk8fI7hCO+d6T0YbQ+oevzfq9dtV1YtHHddif4VSvde19+KvHN2LJnMtw5ZfPMd1n0eZaAMCBlh5MnnQaDnX0jespc9vrn9g+7tiYwr++GsGtV0w/uS3Mfc/dCo12r4uP9kd7on1Q1oSLz/uc8wnySNoSvFJqBMAdANYA2AfgTaVUhYjcJiK3GfvsA7AaQBmAXQAWK6WC0VJFrgtzQImJVS+NjGaWl9iI1xWliTe35n7wwjb87M3k6xWMT1vy53oGR7B+fyv+7bViOBkeB4ZHccyBqhAzXt7x6drV1CpLA52UUkUAihK2LUj4+zEAjzmXNAqbMQX8+JUI/u275+OyvLP8Tg5l4YeLdyJyqMuRu6MgcqpIEvTCDacqINuSNaa29w5i3b4W3P47+1UNFBzFh7oQsTEHT6pCcrDDX/b3PEFrO0jEAK+ZoH7fPihrwuUPrcNwgBYkjufF5xYr7R2KG6vw2021HhzZHifm34lVjbgSAIP6JQ+gnJ2LRte6uaB1VZy9cDsqm7oxedJp6OwbQveJYXz+9E/5naykvPhe/PztspOPY4261a297h/YQ258jmG9ZkdGxzB5kj9laZbgbVBKudbwFCSZdQ0zL1jtqO10fBKm3sERT1eBWrK1zvUfzoqmblff3012A68fgdpuXXk2pzs+f6vKj+LP7l2FAy3+LMSdswH+zYj90a+v7zyMr/+/tajJsrQ177299gOUhwXz6+ZvTfl8shXpnQqCK0qbsL85ecBr7PJ2Oob7VlZiY1XqgXkflEV7z6QaAR2wmytL2noGUR7imUkz7bETO1Wx7/r8DTUoPtRp+33WVka7W/r1GWod4JNdT0ePn8BHGUxFsNF4TX2WK7+/ZmPBDLulHT9iSOJFlG0J7c439mDm0x9n/Po1Fc3YdMDeSOlY8E2W9nSrVcVK4F39yQfHZXpu/OypMeuZzfjb51IPfgtC1Um2i9Dc+cYe3B1XdRYTy9pja6rwgxe2Z3UMP2gZ4NN94az3dRYcOXYi6/QA0VXj8woKcTxFACBnVB7txs0v7bL1mlgQ9WtUblC194ajSjLbCeJWlDZhWUS/9YG1DPC9Rp3vvqPZ12u29WQ+N0u8RR9He0s0uF29ENBqADfrsEfHApppnygVHeE64PcauS7J5Gwn+/pl+80JerWblgH+k8PH/E6C1tIF61gp+MTQxACTrP4+G4+s3u/4eyYK+oUMjL9zfXRNFS6at9r0HHjJyeqlWPbsnItk3TVj7zXn5d3uNtjHjm/8+fzGGmw72O7e8RLkbDdJXQ352M88vh67obMfVzy6wdXjxHxc7d0Fk6mhkTF3VxVK+EzeMqobYvMjeS/5D7n59LTpo3YmZQO/K9xihZ1YoejR1VUAnJk/yQotS/BeGBtTuPONPYjU229Zd9PQSDAGEtVl2RDtNbdL6F+etwqlLvSkcPqGyO9xFEFosAXCccdmhXYBfmR07GR9t5u6B4axorSJy+Mh+PNx2JFpgEkXEHQJGGZSNUyHPt8hvGuIp12AXxZpQGeG67SaCdLJsiLs11Mm/C51us3N7DnSicDkIglKSTzW/h7JoA+7k/z6hmoX4K30Uc/2gkn38nnvV2R3gCTH/HXRPhxsc35I+0f7W9A7mHld7cl6xpNdDSd+RmEPwenuUoIS0OyI1HfisofW4f0SJxeoyc7d75Q7uuhKlzHyfDiua/T2gx2oTjOy1Km70pPfC58uAO0C/KKP6xx9v/gL14+LOHbIxq5+/HZzLW5Zsjvl/nZLs3XtfbhlSQQ/f9va3OSp3LP8VF/kZOnwIw42dPajZyD1+INsr7+gxve3UozYjnUj3u1iO1L85xr7TqS7jpxsvzE71o2LdqApyY9I4mR46cZFpP3hT50812kX4N20bl9mC3FvP9iR9VwjsXjpdJ/vPqPkXt+eff/8dKM9nWL3DuyKRzeYzpC4fl8LCsuOGu8ZCz7ml2RHSAb8JPKiC6kZs0/xYFs0cGd/B+1ecXidMbWALhjgM/BBWRPuX1lpef+5ryVviF2ytQ4X3/+hE8kKtXRDzUsajmEsix+3Ay0Tq7ZufSWC239vbe76X62owOEOb+fAibE7DN/r2gCrpdRsCydejjJ2qt2jzOg5NepTOxEDfAoi5l+q90ua8K6xsHa21Tb3razE8RPBm77gqbUHMu63LSK2g0yqoeY7ajtw/fytWJikd5RX107jMX8C/IOF+9Lu0zM4Mq5azOmPpKXbvcXIgyzb67vKqOv36/NjgNeMU8HumfXVeK/E2vqibjvSFZ0P6IAxd3riRRe7Zc8m63Zf29E76MhUGE4aGLb2g/wPz281nVgrFS/HNQS9wbqhsx/vFEfbNsyuN7P0+9XRiwE+S8f6h7HYg373Vo1l+E0ye1VWg6YS3tDNL/iBll7kFRSiti37ILTZ4iyUM5/5GLOeyXzGSz99cvhY0om13DxPJ3tZBTyAp3P9/K34v29l3ynBC9oH+L1Hshs9aOXL+GDhPgyOmDcwbqvxbhh9XkEhth3ssPUary82J46XLAhtqMqsEdz6gU89TOw/Hp+vDyua3Z2WIMfZmovGhXr7DgfH2bhN+wD//d9MnMvajVb4ZF+kmxbvdOT9g9qPvKtvaML8NyLODKApazwGIC7vDl2rxYc60dDpbH16/Pmf+1oxFmw66Oj7Jz/uRCUeT7aXcgI5i9E4frdk7+ZFYaS1Z9BWoSxV7uLXJOBAJxv2He3G6r3NfifDE07Pvjg6pvDgB5Vo7cm80Sf+y/qNB9aixGSJv5+/Y6+O18zfPRddWepkF0aHIvwPXtie8URoI6Nj6E7Tpx4Ajhzzr1HynU/srVb2+o7DuGnRjoyPZ3ZWTs7imPG7JudmsPzViopxhbJsvnF21yRwQyhnkwxd3WeAit9batqxeEsd6jv6sPjmyzJ+n4bOfkw6zfrXP4g9hcaxcI4auvqT3pGFvV7ZbtVeOnZ+jK3eTWXyEWc+t5DDF61PrayhDPBh05PFNABOi/UlH0nok2z3C+3WVMB+zV9updquqtn5aSLCwon4lOw9rnh0A264dNrJv528a832nRKTkqytLahCWUVjl1cr29itQnjuo2qXUmJNRdNxjLnQFpjq+hwaGUsaxBs6+/GVX66esP3UAshpjmshbbHFsZ02MXgF6LYtYNIF8PbeQfy6aJ8jo7advrN64IP0AxzN4sAmn9YsyIkAH9Qqncc/PODbsWtae3Hts1vw6Bpvh7Jf+fhGfOWXq/FWpAFXPjb+LiBpX+vYQiIn/zS/8PsslP5f2VZvMaVmyZh43JHRMbT3OrOsYyqJk8xVt/RMWC8421hmtaokr6AQeQWF2JDBwvVWFLxTht9urvV05aNkXtxSh6rmUxOTHbBwF7e6onnCKlGlJu1UXsiJAJ/pII2BodGTs9EF3YcV1hqdY0Gq0Rg8VJbBIhTpAknsvc30G0H4v98uQ71PQ/8TDQyPjptt0I77V1Yi/8F1J+f0cctVT2zCxrhuoH/91GZ8++GPku6fOGkWAGypbk85U2Pxoa6kz5mVhGM9hayWkq1W88TORWIBftjY8FBhJV7e6uykgskMDI9h1jObPTmWG1gHn4JTXRzd1to9gLmvFVva98SQ9TqZuvZeNHT247yz/mjc9nTXqZPtSXe+sQdfm/ZZ597QxEXzJlYLmTHL1yqjN1d/QjWgG21q1S296OofwncvPMf0+fhD9gxM/MH54Ys7cfbpU1Iew4lkx99ZZPI5JPvBiHW9LSpvRlF5M+Z8e7rpfjWtPfizc86wf+Ak4n9odmUx86Yfi8PnRAk+kW7rQ8y20cVtaNR6e8Sij+tca0w1Y3YBrChtwqvb7U22lcyOWmd7isRLjElKIavJ0cw0dvXjp8tK8e+vW/sxN9MeNyvmnoT+8n1DI7j22eyrM7/98Efmw/Udbpd4b4/5PPbfezL7ErflGGEjS/Hp9WognKUALyIzRaRKRGpEpCDFfpeJyKiI/KNzSbQm3bD6d/c0hnrln1S//o2dyatE0glS974HC80bsBKHuNe0ZtabZfbCzPt6J2d+XpZFGnD+PUWOHik2HfPOOmfmb1+eECCL67uwvzn1QhjxYufDaueCxRbXaug3aUsx62b7k2Ulad8rVXVhKm5Eivi7vJ+96c1UB2kDvIhMAjAfwCwAMwDcKCIzkuz3CIA1TifSinQDNX66rBTvlzRhuc1BIH7LNv6mXegiQL95B9PMJRMLJHctLfEgNfak+qF06tb8zRSLd7jNbKrkVIG92mR65neTlLgB4K3iiXmLL5BlMqX26JiyNBOnV1bErZy1otSbifyslOAvB1CjlKpVSg0BWArgOpP9/hPAOwBcnhDEXCRFA1HMT5aV4Gdvllra1y4/SsINnf3IKyicMFVAvJlPB7MHkR0NWdyhOCnVHWCyGN7RO4jN1dYmMMuE0yOdk0k2MjlS34kfvjixrWq7i9VhViX+sBb7vC7r7nrn4046VgL8uQDip55rNLadJCLnAvh7AAtSvZGIzBWRiIhE2trc+9KncyTD27agKTXmaknlyLETqMxyNSmKSjXjZrJS+tVPf4w5L6deZjGsRFKXygHzH0WrVaUK0XYHp9y0KBydJpxkJcCbFRESz9DTAO5WSqVswVNKLVRK5Sul8qdOnWoxic57ap1//c+TaT4+4Nqc23cu3XPyceK1dXLOkBTX3LH+Id8HZQH+txeYfUap7p4AuN5HvsPB9288Zq/gM5Jh11Kz1bXMLNlaj798ZAMqmrKbETaXWekm2QjgvLi/pwFIrEDKB7DUuF08G8A1IjKilHrPiUSGxcosRkl+69frAQD1D1+bdt/69j7UtjszbN5KzLz3vb0n1y71UyYB/qP949fYDFKbgxPiewalK02ns8tm4+2u+k5c8IXTU+5zX8LSls3HByw3YO6si+Yt05k/s+m149Xod7dZKcHvBnCBiEwXkSkAZgNYEb+DUmq6UipPKZUH4G0A/5FrwR2ILv7hhSsf34hblkScmR/Ewj5uD+Kxo9Xm0me3vpJ8PVy73FzsOazs9lL5i4fXe7ayajbXx/MbrU33HPTvRNoSvFJqRETuQLR3zCQALymlKkTkNuP5lPXuFH4dveNH83q5fFuibxp3OlYs3HzQ0RK72Xvpdkdgl90ffzuf16l97f8kVLf0YKsHi+0E/fxbGsmqlCoCUJSwzTSwK6X+JftkUcz7DqyLmm3///KEVbG8ulOZSGxdUP9b5P48OwG/vgPJ7hiiTKrm/vqp8E4v4KScHMkaJokNwm41NMZWfw8yvxtZzWS1bm2OCvOAw3h+TW1th1YB3q3Z7YIqk8skMUgGMGYGVhDDkiax0lzA83bzy7tcGVPjJK0mG/uwMvjL+KVbMGBsTOF/i6yNvntklbUqiPj1URMDQt9g8EshTqrIYkyA1sHUQ1YHZ40ZH7hTc/oMjoxNmMY3G3Z7HflBqxK8n6yWhFeWpu5uuKehC4u3WJuzI3E+8GS6TWYWjEnXjztInLjbOKFJ97eYMObHbhWND5MwakOrAJ9N6SxbVkslqebjBtz/MgexHtsqv9MePx97UNy/Mv0KQ0Fjt5H1ybVVbiVFe1oF+EwWr/DaaosLc7jlF8vLfT1+mB1N8+NM1lidKiRW0k83CR0lp1WA91NnnzMrP7ldzzswHJ4qGdJTiU/L1+UiBniHvOPQNMRudSG7+aVdoR9+bXdRcwo3nevevapuZIDPEZsOtE1YwYeI/OFVj6zwdZNcVYClUzb6nYoJ/mTPH+HrU7Kf2vQraz6DpVOSNxZ/auA0DE7JrJol74NPY+mU8NZnnlExGddOCc68OASc0TkZPTwntlWOfQlA+okFs8USfMCk+2EfzGLkZH1HeIM7APQEaNIziuI5CbbwleBnPYzZm5wbrOCUgm9chIctDjxK5fd/882cXJiAKNfc4sExWIIPmDvf2JN+JyIiCxjgHTLoUPfD9l5nulsSETHAOySIywASUW4LXYC3Ov8KEVGuC12Af3ad/4s/ExGFQegCPBERWcMAT0SkqdAF+HQLZhARUVToAnyHQ7M2EhHpLnQBnoiIrAldgD/N72V9iIhCIoQB3u8UEBGFQwgDPCM8EZEVoQvwjO9ERNaELsATEZE1oQvwwiI8EZEl4QvwfieAiCgkQhfgSxuP+Z0EIqJQsBTgRWSmiFSJSI2IFJg8/08iUmb82yYiFzuf1KiW7kG33pqISCtpA7yITAIwH8AsADMA3CgiMxJ2qwPwXaXU1wA8AGCh0wklIiJ7rJTgLwdQo5SqVUoNAVgK4Lr4HZRS25RSXcafOwBMczaZRERkl5UAfy6Ahri/G41tydwKYJXZEyIyV0QiIhJpa2uznkoiIrLNSoA367iiTHcU+StEA/zdZs8rpRYqpfKVUvlTp061nkoiIrJtsoV9GgGcF/f3NABNiTuJyNcALAYwSynV4UzyiIgoU1ZK8LsBXCAi00VkCoDZAFbE7yAifwJgOYB/VkodcD6ZRERkV9oSvFJqRETuALAGwCQALymlKkTkNuP5BQB+CeDzAJ43RpqOKKXy3Us2ERGlY6WKBkqpIgBFCdsWxD3+MYAfO5s0IiLKRuhGshIRkTUM8EREmmKAJyLSFAM8EZGmGOCJiDTFAE9EpCkGeCIiTTHAExFpigGeiEhTDPBERJpigCci0hQDPBGRphjgiYg0xQBPRKQpBngiIk0xwBMRaYoBnohIUwzwRESaYoAnItIUAzwRkaYY4ImINMUAT0SkKQZ4IiJNMcATEWmKAZ6ISFMM8EREmmKAJyLSFAM8EZGmGOCJiDTFAE9EpClLAV5EZopIlYjUiEiByfMiIs8az5eJyCXOJ5WIiOxIG+BFZBKA+QBmAZgB4EYRmZGw2ywAFxj/5gJ4weF0EhGRTVZK8JcDqFFK1SqlhgAsBXBdwj7XAXhVRe0A8DkR+aLDaSUiIhusBPhzATTE/d1obLO7D0RkrohERCTS1tZmN60AgF33XpXR64iIguKeay7y5DiTLewjJttUBvtAKbUQwEIAyM/Pn/C8Feec8Yeof/jaTF5KRJRTrJTgGwGcF/f3NABNGexDREQeshLgdwO4QESmi8gUALMBrEjYZwWAHxm9ab4F4LhS6qjDaSUiIhvSVtEopUZE5A4AawBMAvCSUqpCRG4znl8AoAjANQBqAPQDmONekomIyAordfBQShUhGsTjty2Ie6wA3O5s0oiIKBscyUpEpCkGeCIiTTHAExFpigGeiEhTEm0f9eHAIm0ADmX48rMBtDuYnDBgnnMD85wbssnzl5RSU63s6FuAz4aIRJRS+X6nw0vMc25gnnODV3lmFQ0RkaYY4ImINBXWAL/Q7wT4gHnODcxzbvAkz6GsgyciovTCWoInIqI0GOCJiDQVugCfbgHwoBORehEpF5ESEYkY284SkbUiUm38f2bc/r8w8lolIlfHbb/UeJ8aY8FzMbZ/SkSWGdt3ikieD3l8SURaRWRv3DZP8igiNxvHqBaRmz3KcrI83yciR4xzXSIi18Q9p0OezxORDSKyT0QqROQuY7uW5zpFfoN7npVSofmH6HTFBwGcD2AKgFIAM/xOl8081AM4O2HbowAKjMcFAB4xHs8w8vgpANONvE8yntsF4C8QXU1rFYBZxvb/ALDAeDwbwDIf8vgdAJcA2OtlHgGcBaDW+P9M4/GZPub5PgD/ZbKvLnn+IoBLjMdnADhg5E3Lc50iv4E9z2ErwVtZADyMrgPwivH4FQDXx21fqpQaVErVITrf/uUSXdD8M0qp7Sp69l9NeE3svd4GcFWsdOAVpdRmAJ0Jm73I49UA1iqlOpVSXQDWApjpdP7MJMlzMrrk+ahS6hPjcQ+AfYiuxazluU6R32R8z2/YArylxb0DTgH4UESKRWSuse0LylgBy/j/HGN7svyeazxO3D7uNUqpEQDHAXzehXzY5UUeg/j9uENEyowqnFhVhXZ5NqoSvgFgJ3LgXCfkFwjoeQ5bgLe0uHfAfVspdQmAWQBuF5HvpNg3WX5TfQ5h+4yczGPQ8v4CgD8F8HUARwE8YWzXKs8icjqAdwD8RCnVnWpXk22hy7dJfgN7nsMW4EO/uLdSqsn4vxXAu4hWO7UYt20w/m81dk+W30bjceL2ca8RkckAPgvrVQdu8iKPgfp+KKValFKjSqkxAIsQPdeARnkWkT9ANNj9Tim13Nis7bk2y2+gz7ObjRIuNHJMRrRxYTpONbJ+1e902Uj/pwGcEfd4G6L1aI9hfKPUo8bjr2J8I00tTjXS7AbwLZxqpLnG2H47xjfSvOlTXvMwvsHR9Twi2gBVh2gj1JnG47N8zPMX4x7/FNH6WG3ybKTxVQBPJ2zX8lynyG9gz7PnF74DH/I1iLZeHwRwr9/psZn2840TXgqgIpZ+ROvY1gOoNv4/K+419xp5rYLR0m5szwew13juOZwalfyHAN5CtEFnF4DzfcjnG4jeqg4jWvK41as8ArjF2F4DYI7PeX4NQDmAMgArEgKBDnn+S0SrCcoAlBj/rtH1XKfIb2DPM6cqICLSVNjq4ImIyCIGeCIiTTHAExFpigGeiEhTDPBERJpigCci0hQDPBGRpv4/b1AXmxdKmhcAAAAASUVORK5CYII=\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"code","metadata":{"id":"6gQxseWHP6MA","outputId":"41643246-c704-4c20-8527-6bccbc922659"},"source":["distances_mean_non_zeros = distances_nei.mean(axis=1)[distances_nei.mean(axis=1) != 0]\n","distances_mean_non_zeros\n","plt.plot(np.sort(distances_mean_non_zeros))\n","plt.plot([0,250_000], [np.percentile(distances_nei, 99.5), np.percentile(distances_nei, 99.5)])\n","\n","anomalous_distance_threshold = np.percentile(distances_nei, 99.5)"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXgAAAD4CAYAAADmWv3KAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAXnUlEQVR4nO3de3BcZ3nH8e+zu9LKsuSrJMfxJbaJc3EKIbFw0ubSpEwa20wJhbaTAAUCjCfThKEw7ZByaekwHaAMA6Uk8RgmDaGFkJIALnVI0wKBITc7IRcbY0fYsS3bseTY8UWyLrv79I89klfrtbSSd3X2HP8+Mzt79t1Xu8+rTX5+9Z6z55i7IyIi8ZMIuwAREakOBbyISEwp4EVEYkoBLyISUwp4EZGYSoX1xi0tLb5o0aKw3l5EJJKeffbZg+7eWk7f0AJ+0aJFbNq0Kay3FxGJJDPbVW5fLdGIiMSUAl5EJKYU8CIiMaWAFxGJKQW8iEhMKeBFRGJKAS8iElMKeBGRKvnuM7v5/rOdob2/Al5EpEoe3LSHHz2/N7T3HzPgzexeM+sys82ned7M7Gtm1mFmL5rZ5ZUvU0RExqucGfx9wMpRnl8FLA1ua4B7zrwsERE5U2MGvLv/Ajg0SpebgPs97ylghpnNrVSBIiIyMZVYg58H7Cl43Bm0ncLM1pjZJjPb1N3dXYG3FhGR06lEwFuJtpJX8nb3de7e7u7tra1lne1SREQmqBIB3wksKHg8H9hXgdcVEZEzUImAXw+8Lzia5krgiLvvr8DriojIGRjzgh9m9l3gOqDFzDqBfwDqANx9LbABWA10AL3ArdUqVkREyjdmwLv7LWM878DtFatIREQqQt9kFRGJKQW8iEhMKeBFRGJKAS8iElMKeBGRmFLAi4jElAJeRCSmFPAiIjGlgBcRiSkFvIhITCngRURiSgEvIhJTCngRkZhSwIuIxJQCXkQkphTwIiIxpYAXEYkpBbyISEwp4EVEYkoBLyISUwp4EZGYUsCLiMSUAl5EJKYU8CIiVeIe7vsr4EVEqsjMQntvBbyISJWEPIFXwIuIVI074c3fFfAiIlXjQIgrNAp4EZFq0gxeRCSGInEUjZmtNLNtZtZhZneWeH66mf2Xmb1gZlvM7NbKlyoiEi2O1/ZRNGaWBO4CVgHLgFvMbFlRt9uB37j7pcB1wJfNrL7CtYqIRIp77S/RrAA63H2Huw8ADwA3FfVxoNny/1Q1AYeATEUrFRGJGPfa38k6D9hT8LgzaCv0deBiYB/wEvBRd88Vv5CZrTGzTWa2qbu7e4Ili4hESQ0v0VC6uuJdBzcCzwPnAm8Gvm5m0075Ifd17t7u7u2tra3jLFVEJFqi8EWnTmBBweP55GfqhW4FHva8DmAncFFlShQRiSZ3r/klmo3AUjNbHOw4vRlYX9RnN/BWADObA1wI7KhkoSIiURTmTtbUWB3cPWNmdwCPAkngXnffYma3Bc+vBT4H3GdmL5Efzyfc/WAV6xYRiYQwZ/BjBjyAu28ANhS1rS3Y3gf8cWVLExGJtkh80UlERMbPcazGj6IREZEJiMJx8CIiMgE6m6SISIxpiUZEJIY85L2sCngRkSpxCPVAeAW8iEi1ROBskiIiMkE1fT54ERGZmCicbExERCbA3bVEIyISRzoOXkQkpqJwyT4REZkg7WQVEYkhD3k3qwJeRKRKtEQjIhJT7uibrCIicaWTjYmISMUp4EVEqsTddRy8iEgchbwEr4AXEakWXbJPRCTGtJNVRCSG9EUnEZGY0hKNiEhM6WySIiKxpjV4EZHY8ZAv6aSAFxGpGn3RSUQkliJxNkkzW2lm28ysw8zuPE2f68zseTPbYmaPV7ZMEZHoCXsna2qsDmaWBO4CbgA6gY1mtt7df1PQZwZwN7DS3XebWVuV6hURiZRa/6LTCqDD3Xe4+wDwAHBTUZ93Aw+7+24Ad++qbJkiItHjIe9lLSfg5wF7Ch53Bm2FLgBmmtnPzexZM3tfpQoUEYmqml+iofQ+guJ/llLAcuCtwBTgSTN7yt23j3ghszXAGoCFCxeOv1oRkQiJwk7WTmBBweP5wL4SfX7i7j3ufhD4BXBp8Qu5+zp3b3f39tbW1onWLCISGRbiFL6cgN8ILDWzxWZWD9wMrC/q8yPgGjNLmVkjcAWwtbKliohES9hr8GMu0bh7xszuAB4FksC97r7FzG4Lnl/r7lvN7CfAi0AO+Ka7b65m4SIitS7kL7KWtQaPu28ANhS1rS16/CXgS5UrTUQk4nQ2SRGReMpfsq+21+BFRGSCNIMXEYmhsHeyKuBFRKokv0QTHgW8iEiV6JJ9IiIxVutfdBIRkQnwkI+EV8CLiFRJFM5FIyIiE+AQasIr4EVEqsX1RScRkdjSUTQiIjGknawiIjGlnawiIjEV9iX7FPAiIlWknawiIjGkk42JiMSUlmhERGJKO1lFROJMJxsTEYknzeBFRGIm7B2soIAXEamKoXzXTlYRkZgZmr/rOHgRkZjSDF5EJGa0Bi8iElMnl2jCo4AXEakC7WQVEYmpoXPBm77oJCISL7lc/j6hgBcRiZfBIOFTiRoPeDNbaWbbzKzDzO4cpd9bzCxrZn9WuRJFRKInk80v0dSnwptHj/nOZpYE7gJWAcuAW8xs2Wn6fRF4tNJFiohEzWA2mMEna3sGvwLocPcd7j4APADcVKLfR4CHgK4K1iciEkkDmXzA1yVreAYPzAP2FDzuDNqGmdk84E+BtaO9kJmtMbNNZrapu7t7vLWKiERGJhcs0dR4wJf6+6L4K1pfBT7h7tnRXsjd17l7u7u3t7a2llmiiEj0DC3RhDmDT5XRpxNYUPB4PrCvqE878EBwvGcLsNrMMu7+w0oUKSISNUNLNGGuwZcT8BuBpWa2GNgL3Ay8u7CDuy8e2jaz+4AfK9xF5Gw2NIMPc4lmzIB394yZ3UH+6JgkcK+7bzGz24LnR113FxE5Gw2twdf6Eg3uvgHYUNRWMtjd/QNnXpaISLQNDh9FU9uHSYqIyDgNDB8HX9tH0YiIyDgN7WRN1/I3WUVEZPx6B/JHjTfWJ0OrQQEvIlIFPQMZAKamy9rVWRUKeBGRKujt1wxeRCSWhmbwjfWawYuIxErvQJaGugTJWj8fvIiIjE9Pf4apIc7eQQEvIlIVvQNZGtPhrb9Dmd9krSmP3AmvvhR2FSIio7rtwDH6B7Pwb18+9clz3girvlD1GjSDFxGpglzOSYS4/g5RnMFPwr96IiJn6pN3/4qmdIpv33pFaDVoBi8iUgW9/VntZBURiaOegUzoO1kV8CIiVdA7oBm8iEgs9fRrBi8iEjv9mSz9mRzNIZ5oDBTwIiIV13W0H4C25oZQ61DAi4hUWNexPgDapqVDrUMBLyJSYQeCGfycaZrBi4jEyoGj+Rm8Al5EJGYOHO2nLmnMbKwLtQ4FvIhIhXUd7aOtuQGzcM9Fo4AXEamwA8f6mBPyDlZQwIuIVNz+I32hr7+DAl5EpKIGszn2HOrlvNlTwy5FAS8iUkk7D/YwmHUuOqc57FIU8CIilbR1/1EALpqrgBcRiZXfvnqMuqSxpKUp7FIU8CIilbR1/1He0NpEfSr8eC2rAjNbaWbbzKzDzO4s8fx7zOzF4PaEmV1a+VJFRGqbu7N57xEuOXd62KUAZQS8mSWBu4BVwDLgFjNbVtRtJ/CH7v4m4HPAukoXKiJS63a91svB4wNctnBG2KUA5c3gVwAd7r7D3QeAB4CbCju4+xPufjh4+BQwv7JliojUvmd2HgJgxeJZIVeSV07AzwP2FDzuDNpO50PAI6WeMLM1ZrbJzDZ1d3eXX6WISAQ8vr2btuY0S9vC38EK5QV8qZMpeMmOZteTD/hPlHre3de5e7u7t7e2tpZfpYhIjRvI5Hh8ezd/dFFb6OegGVLO9aQ6gQUFj+cD+4o7mdmbgG8Cq9z9tcqUJyISDb98uZvj/RluWDYn7FKGlTOD3wgsNbPFZlYP3AysL+xgZguBh4G/dPftlS9TRKS2fefp3bQ01XPN0tpZnRhzBu/uGTO7A3gUSAL3uvsWM7steH4t8PfAbODu4E+TjLu3V69sEZHasfu1Xn66rYuPXH9+TRz/PqSsS367+wZgQ1Hb2oLtDwMfrmxpIiLRcP+Tr5A04z1Xnhd2KSPUzj81IiIR1NOf4cFNe1j5e+fUxCmCCyngRUTOwDd+uYOjfRk+fM2SsEs5hQJeRGSCdr3Ww9rHf8fb3jSXNy+YEXY5p1DAi4hMwGA2x8cffIG6RIJPv+3isMspqaydrCIiMtI//fdWnt11mK/dchlzp08Ju5ySNIMXERmnf39qF/c98QofvGoxb7/03LDLOS0FvIjIOPzg15185kebuf7CVj65+qKwyxmVAl5EpEzfeXo3H/veC1y5eDZ3v2c5qWRtR6jW4EVExpDLOf/60w6+8r/buf7CVu5573Ia6pJhlzUmBbyIyCi6jvXxmR9u5tEtB3jnZfP4/LveSDpV++EOCngRkZIy2Rz3P7mLrzy2nf5Mjk+/7WI+dPXimjkVcDkU8CIiRTa+cojP/HAzv331GNde0Mo/vv0SFrdMDbuscVPAi4gEuo/18/lHtvLwc3uZN2MKa9+7nBsvmROpWXshBbyInPW6jvVx369e4dtP7qIvk+X269/A7defT2N9tCMy2tWLiJyBjq7jfPOXO3j4ub0M5nKsvOQc/vbGC1nSWhvXVD1TCngROav09Gf4yeZXeei5Tp743WukUwn+4i3z+fDVS1gUwXX20SjgRST2sjnnqR2v8dCznTyy+VVODGZZOKuRj99wAe+5YiGzm9Jhl1gVCngRiSV3Z/uB4/zo+b384Nd72X+kj+Z0indcdi7vunw+y8+bGdmdp+VSwItIbAxmc2x65TA/39bFY1sPsKO7h2TCuHZpC59cfTE3LJsTiW+gVooCXkQi7cDRPh7f1s3PtnXxy5cPcrw/Q13SuGLxbG69ajE3XjKHtubaupTeZFHAi0ik7D9ygmd2HuLpnYd4ZuchOrqOA3DOtAb+5NK5XHdhG1ed30JTWvGm34CI1Cx3Z9drvTzzSj7Mn9l5iN2HegFoTqdoXzSTP18+n2uWtnLx3ObYr6mPlwJeRGqCu3PgaD8v7T3CS3uP8MKe13mh83Ve7x0EYNbUelYsmsUH/mARKxbP4uK500gmFOijUcCLyKQaCvKXu47R0XWcl7uO0xHcDvUMAJAwuGBOMzcuO4dLF8zgLYtmcn5bk2bo46SAF5GqyOacvYdPlAzy4/2Z4X7Tp9RxflsTN1w8h4vmNvPGedNZdu60yJ8moBboNygiE3a0b5C9h0/kb6+fYM+hXnYHt50He+jP5Ib7tjanWdrWxDsvn8fStibe0NbE+W1NtDalNTOvEgW8iJTU05/h1aN9HDjSx/4jfex7/QT7j/ax//UT7Hs9//hYwUwcIJ1KsHBWIwtmNXL1+S0snZMP8fNbm5neWBfSSM5eCniRs8iJgSwHj/fTfbyfg8f6OXh8gO5j/Rw8nr+d3B4YsYwyZPbUes6Z3sCCWY1cuWQWc2dMYf7MKcybMYV5M6doNl5jFPAiEePunBjMcuTEIEdPZDhyYpDXewc4cmJwxO1wb779cO8Ah3sGOdQzwInBbMnXnNFYR0tTmtamNG+cP4OWpnpam9PMnd7AnGkNzJ0+hbnTG86qb4HGgQJeZBJksjl6B7OcGMjSO5Clpz9DT38mvz2Q3z7ef7L9+NCtL8Ox4fvB/H1fhkzOT/teZjCtoY6ZjXXMaKyntSnNBW3NzJpaz+ymNLOn5sO7pSlNS3M9s6emqU8lJvG3IZOlrIA3s5XAvwBJ4Jvu/oWi5y14fjXQC3zA3Z+rcK0iE+buDGadgWyOwUyOgWyOgYL7waLHQ9v9gzn6Mzn6BrP0ZbIjHvdnsvQN5jgxkH+ubzD/OH8fPBeE+kA2N3aRgXQqQXNDiqnpFFPrUzQ3pDh3xhSaG5ppbkjRlE4xbUodzQ0ppk+pG77NmFLP9KA9oePDhTIC3sySwF3ADUAnsNHM1rv7bwq6rQKWBrcrgHuCe6kBuZyTdSebC27u5HJOJufDz2WyTq6ozyltRa+TG9EHMrnc8HY2l8vfu5PN5sj6ybahnxtRR6n3KepTXHcm5yfD+TSBPZj14fZKqU8lSKcSTKlL0lCXpKEukb9PJWluSNHWnB5ub6xP0VCXpLE+f2uoS9KUTtFYn2Rqwf3UdIqm+hSN6SR1Sc2mpTLKmcGvADrcfQeAmT0A3AQUBvxNwP3u7sBTZjbDzOa6+/5KF/z49m4+9+P8W+ffDkb8seqc0laqnw/381PbSvz16wWNzqn9hl5nZFup1xut3xjvUeZ4cz4yvEuNp1YkE0bSjEQCUokECQvaEgmSCUiakUwO9TFSCSNhRjLYrk8lSNclaGpIUZ9MUJdKkE4mqEsmqE/lb8PbSQvu8/3qh9tL9E8maKhLkE4lSdclaAju65MJzY4lMsoJ+HnAnoLHnZw6Oy/VZx4wIuDNbA2wBmDhwoXjrRWApnSKC+c0F7zoiLuh9ynRVl6/k69nBf1K/ewo/Uq8YGGbcWo/G7XfqYEy2s8aDIdiKpEPxsKgzAdo/pYo6pNKngzQZBk/N7ydHBm8w+85yuskrPTYRKQyygn4Uv8HFs8Jy+mDu68D1gG0t7dPaF65/LyZLD9v5kR+VETkrFLOYl8nsKDg8Xxg3wT6iIjIJCon4DcCS81ssZnVAzcD64v6rAfeZ3lXAkeqsf4uIiLlG3OJxt0zZnYH8Cj5wyTvdfctZnZb8PxaYAP5QyQ7yB8meWv1ShYRkXKUdRy8u28gH+KFbWsLth24vbKliYjImdABtyIiMaWAFxGJKQW8iEhMKeBFRGLKPKTvsZtZN7Brgj/eAhysYDlRoDGfHTTms8OZjPk8d28tp2NoAX8mzGyTu7eHXcdk0pjPDhrz2WGyxqwlGhGRmFLAi4jEVFQDfl3YBYRAYz47aMxnh0kZcyTX4EVEZGxRncGLiMgYFPAiIjEVuYA3s5Vmts3MOszszrDrGS8ze8XMXjKz581sU9A2y8weM7OXg/uZBf3/LhjrNjO7saB9efA6HWb2teDC55hZ2sy+F7Q/bWaLQhjjvWbWZWabC9omZYxm9v7gPV42s/dP0pBPN+bPmtne4LN+3sxWFzwXhzEvMLOfmdlWM9tiZh8N2mP5WY8y3tr9nN09Mjfypyv+HbAEqAdeAJaFXdc4x/AK0FLU9s/AncH2ncAXg+1lwRjTwOJg7MnguWeA3yd/Na1HgFVB+18Ba4Ptm4HvhTDGa4HLgc2TOUZgFrAjuJ8ZbM8MccyfBf6mRN+4jHkucHmw3QxsD8YWy896lPHW7OcctRn88AXA3X0AGLoAeNTdBHwr2P4W8I6C9gfcvd/dd5I/3/4KM5sLTHP3Jz3/6d9f9DNDr/V94K1Ds4PJ4u6/AA4VNU/GGG8EHnP3Q+5+GHgMWFnp8ZVymjGfTlzGvN/dnwu2jwFbyV+LOZaf9SjjPZ3Qxxu1gD/dxb2jxIH/MbNnLX8RcoA5HlwBK7hvC9pPN955wXZx+4ifcfcMcASYXYVxjNdkjLEW//u4w8xeDJZwhpYqYjfmYCnhMuBpzoLPumi8UKOfc9QCvqyLe9e4q9z9cmAVcLuZXTtK39ONd7TfQ9R+R5UcY62N/R7gDcCbgf3Al4P2WI3ZzJqAh4C/dvejo3Ut0Ra5cZcYb81+zlEL+Mhf3Nvd9wX3XcAPyC87HQj+bCO47wq6n268ncF2cfuInzGzFDCd8pcOqmkyxlhT/324+wF3z7p7DvgG+c8aYjRmM6sjH3b/4e4PB82x/axLjbemP+dq7pSowk6OFPmdC4s5uZP1krDrGkf9U4Hmgu0nyK+jfYmRO6X+Odi+hJE7aXZwcifNRuBKTu6kWR20387InTQPhjTWRYzc4Vj1MZLfAbWT/E6omcH2rBDHPLdg+2Pk12NjM+agxvuBrxa1x/KzHmW8Nfs5T/r/+BX4Ja8mv/f6d8Cnwq5nnLUvCT7wF4AtQ/WTX2P7P+Dl4H5Wwc98KhjrNoI97UF7O7A5eO7rnPxWcgPwn+R36DwDLAlhnN8l/6fqIPmZx4cma4zAB4P2DuDWkMf8beAl4EVgfVEQxGHMV5NfJngReD64rY7rZz3KeGv2c9apCkREYipqa/AiIlImBbyISEwp4EVEYkoBLyISUwp4EZGYUsCLiMSUAl5EJKb+H4NA5s4nk71OAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"code","metadata":{"id":"nMchgnZ_P6MB","outputId":"5a445fbb-c4b6-46e9-89ca-e4e5cd9ac2ae"},"source":["anomalous_df = list()\n","\n","start = time.time()\n","\n","for index, row in df_testing.iterrows():\n","    distance, _ = neighbors.kneighbors([np.array(row.tolist())])\n","    if distance.mean() > anomalous_distance_threshold:\n","        anomalous_df.append(index)\n","\n","stop = time.time() - start\n","avg_time = stop/df_testing.shape[0]\n","print(f\"Avg processing time of single sample {avg_time}\")"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Avg processing time of single sample 0.0025146860093266427\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"uVPHJzg4P6MB"},"source":["import csv\n","  \n","with open('[KNN]anonmalous_indexes.csv', 'w', newline='') as f:\n","    write = csv.writer(f) \n","    write.writerow(['index'])\n","    write.writerows([element] for element in anomalous_df)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Qjdgwan0P6MB","outputId":"8f058753-8893-4981-baac-1695ecb7244c"},"source":["print(f\"Discovered {len(anomalous_df)} samples to be suspected as anomalous\")"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Discovered 21310 samples to be suspected as anomalous\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Gm5vgkv2P6MB"},"source":["anomalous_full_df = df.iloc[anomalous_df,:]\n","anomalous_full_df.to_csv (r'C:\\Users\\msmen\\MachineLearning\\anomalous_dataset.csv', index = False, header=True)"],"execution_count":null,"outputs":[]}]}