mining-bge/mining-twitter.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "655b1e69",
   "metadata": {},
   "source": [
    "### Mining dataset Using bge-large-en"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cb3181c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip3 install orjson"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "424dad2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import json\n",
    "from tqdm import tqdm\n",
    "from concurrent.futures import ThreadPoolExecutor, as_completed\n",
    "import random\n",
    "import orjson as json"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e32ddc63",
   "metadata": {},
   "source": [
    "### Load json file containing embedding & texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "20f6ee01",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "691012it [01:20, 8548.87it/s] \n"
     ]
    }
   ],
   "source": [
    "data = []\n",
    "\n",
    "\n",
    "with open('bge-embedding/bge-large-en-embedding/twitter.jsonl') as fopen:\n",
    "    for x in tqdm(fopen):\n",
    "        try:\n",
    "            data.append(json.loads(x))\n",
    "\n",
    "        except:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "47d96b98",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "691012"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ff8fb50",
   "metadata": {},
   "source": [
    "**Dataset Format**\n",
    "\n",
    "- v: contains the embedding vectors with 1024 dimension generated from baai/bge-large-en\n",
    "- text:\n",
    "    - 'left' : original text\n",
    "    - 'en': translated english text\n",
    "    - 'ms' : translated malay text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "40b76f10",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'text': {'left': 'I thought alor setar nya da cukup sedih.. kelantan nya peh ... Kedah n kelantan nya MB mana?',\n",
       "  'en': 'I thought Alor Setar was already quite sad.. Kelantan is worse... Who is the MB for Kedah and Kelantan?',\n",
       "  'ms': 'Saya fikir Alor Setar sudah cukup sedih.. Kelantan lagi teruk... Siapa MB untuk Kedah dan Kelantan?'},\n",
       " 'v': [0.13888011872768402,\n",
       "  -0.47501182556152344,\n",
       "  0.40516266226768494,\n",
       "  0.16995717585086823,\n",
       "  -0.6713352203369141,\n",
       "  -1.0514363050460815,\n",
       "  0.10906267166137695,\n",
       "  0.34273001551628113,\n",
       "  0.152670755982399,\n",
       "  -0.2184249609708786,\n",
       "  0.42360803484916687,\n",
       "  -0.836317241191864,\n",
       "  -0.4082200825214386,\n",
       "  -0.32670679688453674,\n",
       "  -0.6629812717437744,\n",
       "  -0.3168671429157257,\n",
       "  -0.37673184275627136,\n",
       "  0.008131159469485283,\n",
       "  -0.31306037306785583,\n",
       "  0.4005916118621826,\n",
       "  0.9667572379112244,\n",
       "  -0.17604903876781464,\n",
       "  -0.9339383244514465,\n",
       "  -0.38919058442115784,\n",
       "  -0.27883780002593994,\n",
       "  0.4602336883544922,\n",
       "  0.27048981189727783,\n",
       "  0.42183181643486023,\n",
       "  1.1284841299057007,\n",
       "  1.2345982789993286,\n",
       "  -0.3928127586841583,\n",
       "  -0.11699055135250092,\n",
       "  0.13553790748119354,\n",
       "  -0.37155747413635254,\n",
       "  -0.011062958277761936,\n",
       "  0.2526552975177765,\n",
       "  -0.10283797979354858,\n",
       "  -0.6808709502220154,\n",
       "  -0.12397260963916779,\n",
       "  -0.7543399333953857,\n",
       "  0.7143051624298096,\n",
       "  -0.16386453807353973,\n",
       "  0.8971184492111206,\n",
       "  -0.6193539500236511,\n",
       "  -0.7012479901313782,\n",
       "  0.32825297117233276,\n",
       "  0.16811493039131165,\n",
       "  -0.12691889703273773,\n",
       "  0.1538655161857605,\n",
       "  -1.1995104551315308,\n",
       "  -0.05443849787116051,\n",
       "  0.32922235131263733,\n",
       "  0.4554484486579895,\n",
       "  0.2729846239089966,\n",
       "  -0.22380852699279785,\n",
       "  0.15986688435077667,\n",
       "  0.07470261305570602,\n",
       "  -0.32676962018013,\n",
       "  0.058928150683641434,\n",
       "  0.6395062804222107,\n",
       "  0.19862693548202515,\n",
       "  -0.34878623485565186,\n",
       "  -0.1696750968694687,\n",
       "  -1.296430230140686,\n",
       "  0.3720093071460724,\n",
       "  0.059021808207035065,\n",
       "  -0.10568959265947342,\n",
       "  0.3661912679672241,\n",
       "  0.6721508502960205,\n",
       "  -0.4851559102535248,\n",
       "  -0.13120436668395996,\n",
       "  -0.05133550986647606,\n",
       "  0.03659121319651604,\n",
       "  -0.7984415292739868,\n",
       "  -0.11651711165904999,\n",
       "  -0.2279365062713623,\n",
       "  -0.424182653427124,\n",
       "  0.142316535115242,\n",
       "  0.38176167011260986,\n",
       "  0.39550861716270447,\n",
       "  0.7543432712554932,\n",
       "  1.0519410371780396,\n",
       "  0.6145016551017761,\n",
       "  -0.20638714730739594,\n",
       "  -0.6517171263694763,\n",
       "  -0.10510257631540298,\n",
       "  0.11421725898981094,\n",
       "  1.24124014377594,\n",
       "  0.2588427662849426,\n",
       "  0.594173014163971,\n",
       "  -0.7008472084999084,\n",
       "  0.9771915078163147,\n",
       "  -0.20630499720573425,\n",
       "  -0.00748100783675909,\n",
       "  0.5760505199432373,\n",
       "  0.6146409511566162,\n",
       "  0.1717379093170166,\n",
       "  -0.2719385623931885,\n",
       "  -1.0061215162277222,\n",
       "  -0.3949615955352783,\n",
       "  0.40917259454727173,\n",
       "  1.1162930727005005,\n",
       "  -0.009453077800571918,\n",
       "  1.0109634399414062,\n",
       "  -0.7084853053092957,\n",
       "  0.26155874133110046,\n",
       "  0.2466534525156021,\n",
       "  -0.632470428943634,\n",
       "  0.21408313512802124,\n",
       "  -1.0108107328414917,\n",
       "  -0.19903381168842316,\n",
       "  -0.5016675591468811,\n",
       "  -0.1902725100517273,\n",
       "  -0.37140214443206787,\n",
       "  -0.442294180393219,\n",
       "  0.20929554104804993,\n",
       "  -0.32757624983787537,\n",
       "  0.4878082275390625,\n",
       "  -0.6365262866020203,\n",
       "  0.2935105860233307,\n",
       "  0.9004396200180054,\n",
       "  0.2608928978443146,\n",
       "  1.1032347679138184,\n",
       "  0.17027902603149414,\n",
       "  0.5458692908287048,\n",
       "  -0.23630057275295258,\n",
       "  -0.7410487532615662,\n",
       "  0.7082163691520691,\n",
       "  -0.688086986541748,\n",
       "  -0.9892474412918091,\n",
       "  0.22289563715457916,\n",
       "  -0.535946249961853,\n",
       "  -0.14291229844093323,\n",
       "  -0.084645114839077,\n",
       "  0.11279319226741791,\n",
       "  0.9551196098327637,\n",
       "  0.7068294286727905,\n",
       "  0.5632721781730652,\n",
       "  0.8190697431564331,\n",
       "  -1.0735360383987427,\n",
       "  0.20952534675598145,\n",
       "  0.6392496824264526,\n",
       "  -0.005831061862409115,\n",
       "  1.1623564958572388,\n",
       "  -0.5520394444465637,\n",
       "  0.36815395951271057,\n",
       "  0.2818247675895691,\n",
       "  0.05037843436002731,\n",
       "  -0.8414368033409119,\n",
       "  -0.35379916429519653,\n",
       "  -0.31688758730888367,\n",
       "  0.4632590115070343,\n",
       "  -0.039881639182567596,\n",
       "  0.6259133219718933,\n",
       "  0.44958779215812683,\n",
       "  0.07471022009849548,\n",
       "  0.45034798979759216,\n",
       "  0.7700124382972717,\n",
       "  -0.31990206241607666,\n",
       "  -0.047013089060783386,\n",
       "  -0.6464188098907471,\n",
       "  -0.41103559732437134,\n",
       "  0.028749650344252586,\n",
       "  0.515004575252533,\n",
       "  0.217298686504364,\n",
       "  0.1402677595615387,\n",
       "  0.09745065867900848,\n",
       "  0.03073151409626007,\n",
       "  -0.18105289340019226,\n",
       "  -0.7562942504882812,\n",
       "  0.9486685991287231,\n",
       "  -0.008033351972699165,\n",
       "  -0.527254045009613,\n",
       "  0.566929042339325,\n",
       "  -0.3036939799785614,\n",
       "  0.6842717528343201,\n",
       "  0.28102797269821167,\n",
       "  -0.3936246633529663,\n",
       "  0.3203740119934082,\n",
       "  0.149150550365448,\n",
       "  -0.1264847218990326,\n",
       "  -0.1440497785806656,\n",
       "  0.2214840203523636,\n",
       "  0.8113918304443359,\n",
       "  0.00183296378236264,\n",
       "  -0.18814446032047272,\n",
       "  0.6765236258506775,\n",
       "  -0.6257396340370178,\n",
       "  -0.659702479839325,\n",
       "  -0.6207560300827026,\n",
       "  0.09816407412290573,\n",
       "  0.17128752171993256,\n",
       "  -0.8333517909049988,\n",
       "  0.24181899428367615,\n",
       "  0.2802140414714813,\n",
       "  -0.4208354949951172,\n",
       "  -1.3068294525146484,\n",
       "  -0.20424784719944,\n",
       "  0.6197670102119446,\n",
       "  -0.5115042328834534,\n",
       "  -0.6561013460159302,\n",
       "  1.342125415802002,\n",
       "  -0.2791976034641266,\n",
       "  0.5940250158309937,\n",
       "  0.6693618893623352,\n",
       "  -1.1002880334854126,\n",
       "  0.13021042943000793,\n",
       "  0.6713966131210327,\n",
       "  -0.6965306401252747,\n",
       "  -0.11901300400495529,\n",
       "  0.6448580026626587,\n",
       "  1.0472028255462646,\n",
       "  -0.3419736325740814,\n",
       "  -0.6255971789360046,\n",
       "  -0.23096561431884766,\n",
       "  0.3819257318973541,\n",
       "  -0.29354387521743774,\n",
       "  0.30576595664024353,\n",
       "  0.04548163712024689,\n",
       "  -0.005833339411765337,\n",
       "  0.7523950338363647,\n",
       "  0.49752742052078247,\n",
       "  0.7949827909469604,\n",
       "  0.6064956784248352,\n",
       "  0.1261110156774521,\n",
       "  0.12388596683740616,\n",
       "  -0.04879363626241684,\n",
       "  0.6404017210006714,\n",
       "  -0.41837695240974426,\n",
       "  -0.6387300491333008,\n",
       "  -0.32516640424728394,\n",
       "  -0.6980464458465576,\n",
       "  0.42929330468177795,\n",
       "  0.9095197319984436,\n",
       "  0.30471572279930115,\n",
       "  0.5205181837081909,\n",
       "  0.5498710870742798,\n",
       "  0.5955251455307007,\n",
       "  0.3950468897819519,\n",
       "  0.04356503114104271,\n",
       "  0.5431339740753174,\n",
       "  0.4907165467739105,\n",
       "  1.1022274494171143,\n",
       "  1.1156026124954224,\n",
       "  0.4941793382167816,\n",
       "  0.8249328136444092,\n",
       "  0.5188125967979431,\n",
       "  -0.5982415080070496,\n",
       "  -0.8484815955162048,\n",
       "  0.05090272054076195,\n",
       "  -0.02886071242392063,\n",
       "  1.160274624824524,\n",
       "  0.607394278049469,\n",
       "  0.13222269713878632,\n",
       "  -0.8071284890174866,\n",
       "  0.2939969599246979,\n",
       "  0.4289507567882538,\n",
       "  0.46609070897102356,\n",
       "  -0.3326614797115326,\n",
       "  -0.43992331624031067,\n",
       "  0.16287998855113983,\n",
       "  -0.25823724269866943,\n",
       "  -0.051900263875722885,\n",
       "  -0.12409057468175888,\n",
       "  0.23405370116233826,\n",
       "  0.5869248509407043,\n",
       "  -0.5175966024398804,\n",
       "  0.6299018263816833,\n",
       "  0.24718332290649414,\n",
       "  -1.1555546522140503,\n",
       "  -0.4739402234554291,\n",
       "  -0.6819400191307068,\n",
       "  0.06988425552845001,\n",
       "  -0.7455418109893799,\n",
       "  -0.25391626358032227,\n",
       "  0.46262291073799133,\n",
       "  0.8790422677993774,\n",
       "  -0.7382025122642517,\n",
       "  0.5857492089271545,\n",
       "  -0.3316476345062256,\n",
       "  -0.029050573706626892,\n",
       "  -0.008742295205593109,\n",
       "  -0.28673192858695984,\n",
       "  0.4170280992984772,\n",
       "  0.2610127925872803,\n",
       "  0.8890891075134277,\n",
       "  -0.6096963286399841,\n",
       "  0.8540274500846863,\n",
       "  0.1366606056690216,\n",
       "  0.09174607694149017,\n",
       "  -0.5758262276649475,\n",
       "  -0.4883385896682739,\n",
       "  0.17674802243709564,\n",
       "  0.12697818875312805,\n",
       "  0.13030356168746948,\n",
       "  -0.25011152029037476,\n",
       "  0.0649292841553688,\n",
       "  -0.1275055706501007,\n",
       "  -1.010108232498169,\n",
       "  -0.8967171311378479,\n",
       "  0.16071026027202606,\n",
       "  -0.1157497838139534,\n",
       "  -0.24877780675888062,\n",
       "  0.4941941797733307,\n",
       "  -0.2640036940574646,\n",
       "  0.37882792949676514,\n",
       "  0.3812759518623352,\n",
       "  -0.03122270479798317,\n",
       "  0.47671279311180115,\n",
       "  0.3857053816318512,\n",
       "  0.19040007889270782,\n",
       "  -0.41203829646110535,\n",
       "  0.4358772933483124,\n",
       "  -0.21358622610569,\n",
       "  -0.8760811686515808,\n",
       "  0.7491867542266846,\n",
       "  0.7863816022872925,\n",
       "  0.38802570104599,\n",
       "  -0.3131912350654602,\n",
       "  -0.45931702852249146,\n",
       "  -0.6076406240463257,\n",
       "  -0.021781442686915398,\n",
       "  0.2744832932949066,\n",
       "  0.757401704788208,\n",
       "  -0.3305518925189972,\n",
       "  0.5546904802322388,\n",
       "  0.2567756175994873,\n",
       "  -1.4034868478775024,\n",
       "  0.23843762278556824,\n",
       "  -0.37626293301582336,\n",
       "  -0.6056421995162964,\n",
       "  -0.5211466550827026,\n",
       "  -0.3975561261177063,\n",
       "  0.35910412669181824,\n",
       "  0.9752227067947388,\n",
       "  -0.012407086789608002,\n",
       "  0.02793543040752411,\n",
       "  0.4716673195362091,\n",
       "  -0.1365303248167038,\n",
       "  0.22370077669620514,\n",
       "  0.0530131459236145,\n",
       "  -0.39243921637535095,\n",
       "  0.19462992250919342,\n",
       "  0.3064817488193512,\n",
       "  0.5406529903411865,\n",
       "  -0.2113945186138153,\n",
       "  0.26193973422050476,\n",
       "  0.1994677633047104,\n",
       "  -0.9214563965797424,\n",
       "  0.21494629979133606,\n",
       "  0.07328982651233673,\n",
       "  -0.03232603147625923,\n",
       "  0.1639692485332489,\n",
       "  0.30959025025367737,\n",
       "  0.13730396330356598,\n",
       "  0.9367293119430542,\n",
       "  0.17374315857887268,\n",
       "  -0.2606528401374817,\n",
       "  0.06863000243902206,\n",
       "  0.149408757686615,\n",
       "  0.6357877850532532,\n",
       "  -0.7108647227287292,\n",
       "  -0.1955173760652542,\n",
       "  -0.3176039755344391,\n",
       "  -0.6258904933929443,\n",
       "  -0.8943216800689697,\n",
       "  -0.12066301703453064,\n",
       "  0.45867329835891724,\n",
       "  0.36181774735450745,\n",
       "  -1.2193856239318848,\n",
       "  0.8668199181556702,\n",
       "  0.2660699784755707,\n",
       "  -0.38290372490882874,\n",
       "  0.6687273383140564,\n",
       "  -0.45874419808387756,\n",
       "  -0.5303101539611816,\n",
       "  0.5921828150749207,\n",
       "  0.5502691864967346,\n",
       "  0.7647573947906494,\n",
       "  -1.04740309715271,\n",
       "  0.8064575791358948,\n",
       "  -0.6142436265945435,\n",
       "  0.42917492985725403,\n",
       "  0.929305374622345,\n",
       "  0.2415921688079834,\n",
       "  0.5263847708702087,\n",
       "  -0.5230474472045898,\n",
       "  -0.9130795001983643,\n",
       "  0.4693600535392761,\n",
       "  -0.5104895234107971,\n",
       "  0.32051751017570496,\n",
       "  -0.18524014949798584,\n",
       "  -0.4576338231563568,\n",
       "  -0.47235310077667236,\n",
       "  -0.9121659398078918,\n",
       "  -1.1570967435836792,\n",
       "  -0.23574107885360718,\n",
       "  0.6047335267066956,\n",
       "  0.8067843317985535,\n",
       "  0.5323695540428162,\n",
       "  0.519706130027771,\n",
       "  0.20572428405284882,\n",
       "  -0.16117238998413086,\n",
       "  0.5031932592391968,\n",
       "  -0.9054036736488342,\n",
       "  0.7524282336235046,\n",
       "  -0.26466959714889526,\n",
       "  1.2116557359695435,\n",
       "  -0.20644254982471466,\n",
       "  -0.6673540472984314,\n",
       "  0.15322767198085785,\n",
       "  -0.22371576726436615,\n",
       "  -0.2060602307319641,\n",
       "  0.2250794619321823,\n",
       "  -0.2237328737974167,\n",
       "  -0.2846260070800781,\n",
       "  -0.15364015102386475,\n",
       "  0.04086146131157875,\n",
       "  -0.459320068359375,\n",
       "  0.4131641685962677,\n",
       "  0.19034238159656525,\n",
       "  -1.3269342184066772,\n",
       "  0.21029162406921387,\n",
       "  0.25653794407844543,\n",
       "  0.10888947546482086,\n",
       "  -0.2658226788043976,\n",
       "  0.24498511850833893,\n",
       "  -0.4436264634132385,\n",
       "  1.1188067197799683,\n",
       "  0.11715807020664215,\n",
       "  -0.1621038168668747,\n",
       "  -0.40711161494255066,\n",
       "  -0.8209226131439209,\n",
       "  -0.9280276298522949,\n",
       "  -0.48358502984046936,\n",
       "  0.6424573659896851,\n",
       "  0.15902665257453918,\n",
       "  -0.2512289583683014,\n",
       "  0.540280818939209,\n",
       "  -0.8215141296386719,\n",
       "  0.8349524140357971,\n",
       "  0.5647491216659546,\n",
       "  0.5323841571807861,\n",
       "  0.4498385190963745,\n",
       "  0.04924102872610092,\n",
       "  0.8482457399368286,\n",
       "  -0.5614816546440125,\n",
       "  0.5906050801277161,\n",
       "  0.32277411222457886,\n",
       "  -0.0730949193239212,\n",
       "  -0.44603824615478516,\n",
       "  -1.0848292112350464,\n",
       "  0.6425403952598572,\n",
       "  -0.058734212070703506,\n",
       "  -0.4317227602005005,\n",
       "  -0.035530123859643936,\n",
       "  -0.030315624549984932,\n",
       "  0.9948959350585938,\n",
       "  0.8353579044342041,\n",
       "  0.1397949904203415,\n",
       "  0.22929327189922333,\n",
       "  -0.1748899519443512,\n",
       "  0.05463111400604248,\n",
       "  -0.0739828571677208,\n",
       "  -0.3723335266113281,\n",
       "  0.43075108528137207,\n",
       "  -0.1905917227268219,\n",
       "  0.010497632436454296,\n",
       "  0.9879156947135925,\n",
       "  0.02097414806485176,\n",
       "  -0.26474708318710327,\n",
       "  0.05394009128212929,\n",
       "  0.5528947114944458,\n",
       "  -0.4553949534893036,\n",
       "  0.3107393980026245,\n",
       "  -0.45048588514328003,\n",
       "  -0.04883292317390442,\n",
       "  0.2511005103588104,\n",
       "  0.14678707718849182,\n",
       "  0.5143595933914185,\n",
       "  -0.7934552431106567,\n",
       "  -0.28812453150749207,\n",
       "  -0.9974035620689392,\n",
       "  -0.4356386959552765,\n",
       "  -1.0911810398101807,\n",
       "  -0.9532116055488586,\n",
       "  -0.4132095277309418,\n",
       "  0.18411606550216675,\n",
       "  -0.35063186287879944,\n",
       "  0.5634192228317261,\n",
       "  -0.19226868450641632,\n",
       "  -0.48453888297080994,\n",
       "  -0.1692260205745697,\n",
       "  -0.12548032402992249,\n",
       "  -0.24788406491279602,\n",
       "  -0.03981531411409378,\n",
       "  -0.9445452094078064,\n",
       "  -0.23677222430706024,\n",
       "  0.8591470122337341,\n",
       "  0.3355470597743988,\n",
       "  0.7425902485847473,\n",
       "  -0.3653549551963806,\n",
       "  -0.5421708226203918,\n",
       "  0.7977428436279297,\n",
       "  -0.3013140857219696,\n",
       "  -0.5659698843955994,\n",
       "  -0.350182443857193,\n",
       "  -0.227270245552063,\n",
       "  -0.499070405960083,\n",
       "  -0.4034203886985779,\n",
       "  -0.6856868863105774,\n",
       "  -0.10356665402650833,\n",
       "  0.7018004655838013,\n",
       "  0.5590631365776062,\n",
       "  0.2793026566505432,\n",
       "  0.37368568778038025,\n",
       "  -0.33275073766708374,\n",
       "  0.16446968913078308,\n",
       "  -0.14321120083332062,\n",
       "  0.9812541604042053,\n",
       "  0.2657914161682129,\n",
       "  -0.6571747660636902,\n",
       "  -0.49802660942077637,\n",
       "  0.5400207042694092,\n",
       "  0.112238809466362,\n",
       "  0.8512346744537354,\n",
       "  0.01658063381910324,\n",
       "  0.44157683849334717,\n",
       "  -0.7501548528671265,\n",
       "  -0.8165869116783142,\n",
       "  -0.1487904191017151,\n",
       "  -0.49612700939178467,\n",
       "  -0.5990598201751709,\n",
       "  -0.8121660947799683,\n",
       "  -0.32253074645996094,\n",
       "  -0.4127420485019684,\n",
       "  0.8562546372413635,\n",
       "  -0.5451838374137878,\n",
       "  0.15376825630664825,\n",
       "  0.03369339182972908,\n",
       "  -0.8510520458221436,\n",
       "  0.3076624572277069,\n",
       "  -0.07716629654169083,\n",
       "  -0.5334230661392212,\n",
       "  -1.0248329639434814,\n",
       "  -0.16803349554538727,\n",
       "  0.10307139903306961,\n",
       "  0.4457460045814514,\n",
       "  0.16113010048866272,\n",
       "  0.4357472360134125,\n",
       "  0.1556590348482132,\n",
       "  0.025825411081314087,\n",
       "  0.7099646925926208,\n",
       "  -0.552933931350708,\n",
       "  -1.0244635343551636,\n",
       "  -1.0873472690582275,\n",
       "  -0.13931787014007568,\n",
       "  0.6063199639320374,\n",
       "  -0.07905884087085724,\n",
       "  -0.24744385480880737,\n",
       "  -0.14662516117095947,\n",
       "  0.6785485744476318,\n",
       "  -1.0858819484710693,\n",
       "  0.5884196758270264,\n",
       "  -0.18120180070400238,\n",
       "  -0.3247430622577667,\n",
       "  -0.42830562591552734,\n",
       "  -0.5919303297996521,\n",
       "  0.9358779788017273,\n",
       "  -0.3595796525478363,\n",
       "  0.9327954053878784,\n",
       "  -0.109742172062397,\n",
       "  0.039665669202804565,\n",
       "  0.7348780035972595,\n",
       "  0.4205878973007202,\n",
       "  -0.3842942714691162,\n",
       "  -0.4004816710948944,\n",
       "  -0.6313183307647705,\n",
       "  -0.6718575358390808,\n",
       "  0.7767939567565918,\n",
       "  -0.49202707409858704,\n",
       "  -0.0870460495352745,\n",
       "  -0.2706548869609833,\n",
       "  -0.16179539263248444,\n",
       "  0.5929695963859558,\n",
       "  -0.3655308783054352,\n",
       "  0.5110445022583008,\n",
       "  1.0130149126052856,\n",
       "  -0.22447416186332703,\n",
       "  -0.38468077778816223,\n",
       "  -0.8700476288795471,\n",
       "  0.1483353078365326,\n",
       "  0.23421458899974823,\n",
       "  -0.4978465139865875,\n",
       "  -0.592731237411499,\n",
       "  0.12639716267585754,\n",
       "  -0.08920778334140778,\n",
       "  -0.16557084023952484,\n",
       "  0.12579002976417542,\n",
       "  -0.8967018723487854,\n",
       "  -0.921280562877655,\n",
       "  -0.17457614839076996,\n",
       "  0.5398040413856506,\n",
       "  -0.8698945045471191,\n",
       "  0.5658590197563171,\n",
       "  -0.41350996494293213,\n",
       "  -0.7244243621826172,\n",
       "  -0.8500742316246033,\n",
       "  0.8915156722068787,\n",
       "  0.23721668124198914,\n",
       "  -0.005295300390571356,\n",
       "  0.04149148240685463,\n",
       "  0.2987123131752014,\n",
       "  0.07348047196865082,\n",
       "  0.238403782248497,\n",
       "  0.48965927958488464,\n",
       "  0.2498820424079895,\n",
       "  -1.0380738973617554,\n",
       "  0.49550309777259827,\n",
       "  -0.4697522222995758,\n",
       "  -0.6744474172592163,\n",
       "  1.301322102546692,\n",
       "  0.9901936054229736,\n",
       "  -1.070554256439209,\n",
       "  -0.9715906381607056,\n",
       "  0.027393262833356857,\n",
       "  0.3980305790901184,\n",
       "  0.1528165191411972,\n",
       "  -0.1169939711689949,\n",
       "  0.6661715507507324,\n",
       "  0.0008658902952447534,\n",
       "  -0.6581791043281555,\n",
       "  -0.08989845216274261,\n",
       "  0.05637525022029877,\n",
       "  0.12504389882087708,\n",
       "  0.08490884304046631,\n",
       "  -0.07748988270759583,\n",
       "  0.3060881495475769,\n",
       "  -0.9242194890975952,\n",
       "  -0.23450060188770294,\n",
       "  0.6022597551345825,\n",
       "  -0.22154203057289124,\n",
       "  0.004394174553453922,\n",
       "  -0.7028225064277649,\n",
       "  -0.6744524240493774,\n",
       "  -0.6008789539337158,\n",
       "  0.02043181285262108,\n",
       "  0.15074564516544342,\n",
       "  -0.3452485203742981,\n",
       "  0.778801679611206,\n",
       "  0.502652108669281,\n",
       "  0.26887252926826477,\n",
       "  0.6691181063652039,\n",
       "  -0.3155409097671509,\n",
       "  0.09664257615804672,\n",
       "  0.3754652142524719,\n",
       "  -0.07469271123409271,\n",
       "  0.18727725744247437,\n",
       "  0.4154371917247772,\n",
       "  0.261141836643219,\n",
       "  0.5822016000747681,\n",
       "  0.34897854924201965,\n",
       "  -0.5707276463508606,\n",
       "  0.667579174041748,\n",
       "  0.3869118094444275,\n",
       "  -0.07384424656629562,\n",
       "  0.43109509348869324,\n",
       "  0.1323663890361786,\n",
       "  -0.4818311333656311,\n",
       "  -0.5247661471366882,\n",
       "  -0.4613879919052124,\n",
       "  -0.48540231585502625,\n",
       "  -0.3329775631427765,\n",
       "  -0.532665491104126,\n",
       "  -0.5442656874656677,\n",
       "  0.7684289216995239,\n",
       "  -0.8597936034202576,\n",
       "  0.13856488466262817,\n",
       "  0.21924534440040588,\n",
       "  -0.5607810616493225,\n",
       "  0.8546189069747925,\n",
       "  -0.19993481040000916,\n",
       "  0.10021588206291199,\n",
       "  -0.830021858215332,\n",
       "  -0.09393035620450974,\n",
       "  -0.4829051196575165,\n",
       "  -0.7795401215553284,\n",
       "  0.38563305139541626,\n",
       "  -0.45948323607444763,\n",
       "  -1.0015075206756592,\n",
       "  -0.2805275321006775,\n",
       "  -0.4417594075202942,\n",
       "  0.2668052911758423,\n",
       "  0.24238790571689606,\n",
       "  0.1301436722278595,\n",
       "  0.20598842203617096,\n",
       "  -0.8070117831230164,\n",
       "  0.22963941097259521,\n",
       "  -0.2816883623600006,\n",
       "  -0.0732535794377327,\n",
       "  -0.08136454969644547,\n",
       "  0.8243277072906494,\n",
       "  -0.7466804385185242,\n",
       "  0.86822110414505,\n",
       "  0.009725983254611492,\n",
       "  -0.3545103073120117,\n",
       "  0.3842042088508606,\n",
       "  -0.1890089064836502,\n",
       "  0.41094210743904114,\n",
       "  0.11478979885578156,\n",
       "  -0.3843706548213959,\n",
       "  -0.7333871722221375,\n",
       "  0.4976820647716522,\n",
       "  -0.5620104670524597,\n",
       "  1.0785856246948242,\n",
       "  -0.16557101905345917,\n",
       "  0.7887964248657227,\n",
       "  0.32952457666397095,\n",
       "  -0.27321699261665344,\n",
       "  1.1603037118911743,\n",
       "  0.3754064440727234,\n",
       "  0.16794466972351074,\n",
       "  0.2171557992696762,\n",
       "  0.26554304361343384,\n",
       "  0.854487955570221,\n",
       "  -0.0879860669374466,\n",
       "  -0.696427047252655,\n",
       "  0.0893724113702774,\n",
       "  0.7467426061630249,\n",
       "  1.1546636819839478,\n",
       "  -0.9192137122154236,\n",
       "  0.26088446378707886,\n",
       "  0.5346505641937256,\n",
       "  -0.24343889951705933,\n",
       "  0.6146345138549805,\n",
       "  -0.835990846157074,\n",
       "  -0.39407268166542053,\n",
       "  -0.5161464810371399,\n",
       "  -0.029338819906115532,\n",
       "  0.4530507028102875,\n",
       "  0.8474752902984619,\n",
       "  0.4634039103984833,\n",
       "  -0.2752656042575836,\n",
       "  -0.520357072353363,\n",
       "  -0.28909507393836975,\n",
       "  -0.39275187253952026,\n",
       "  0.5041342377662659,\n",
       "  -1.309630036354065,\n",
       "  -0.01496975775808096,\n",
       "  0.4326956570148468,\n",
       "  -0.3670022487640381,\n",
       "  -0.362544447183609,\n",
       "  -0.5398098230361938,\n",
       "  -1.2329857349395752,\n",
       "  -0.025777041912078857,\n",
       "  0.27545255422592163,\n",
       "  0.17018282413482666,\n",
       "  -0.2894562780857086,\n",
       "  0.8068792819976807,\n",
       "  -0.12486567348241806,\n",
       "  0.6222126483917236,\n",
       "  -0.04562254995107651,\n",
       "  -0.027024012058973312,\n",
       "  0.33652693033218384,\n",
       "  0.8328700065612793,\n",
       "  -0.9824222326278687,\n",
       "  -0.7243567705154419,\n",
       "  -0.22825443744659424,\n",
       "  0.5291345715522766,\n",
       "  0.2572166621685028,\n",
       "  0.8923026323318481,\n",
       "  -0.1385796070098877,\n",
       "  -0.36142274737358093,\n",
       "  0.35240498185157776,\n",
       "  -0.2666386067867279,\n",
       "  -0.8708460927009583,\n",
       "  -0.0408158153295517,\n",
       "  -0.1883751004934311,\n",
       "  0.4864000678062439,\n",
       "  -0.620879054069519,\n",
       "  0.2919556200504303,\n",
       "  -0.6685084700584412,\n",
       "  0.6849991083145142,\n",
       "  0.6375516653060913,\n",
       "  -0.5357447266578674,\n",
       "  -0.2535194158554077,\n",
       "  -0.2542037069797516,\n",
       "  0.6903282403945923,\n",
       "  -0.7909750938415527,\n",
       "  0.0817243903875351,\n",
       "  0.6131774187088013,\n",
       "  0.40484294295310974,\n",
       "  0.22195035219192505,\n",
       "  0.9414003491401672,\n",
       "  -0.5363695025444031,\n",
       "  0.7139204144477844,\n",
       "  0.18352386355400085,\n",
       "  -0.24554508924484253,\n",
       "  -0.45181870460510254,\n",
       "  -0.2154500037431717,\n",
       "  0.8496065139770508,\n",
       "  0.13795354962348938,\n",
       "  -0.46282830834388733,\n",
       "  0.9743940234184265,\n",
       "  0.20066408812999725,\n",
       "  -0.03772677481174469,\n",
       "  -0.23276998102664948,\n",
       "  -0.25097987055778503,\n",
       "  0.3321523070335388,\n",
       "  0.53218674659729,\n",
       "  -0.186421200633049,\n",
       "  0.6235296130180359,\n",
       "  -0.11806854605674744,\n",
       "  0.47354382276535034,\n",
       "  -0.05176375433802605,\n",
       "  0.2590816915035248,\n",
       "  0.7668874263763428,\n",
       "  -0.6575620770454407,\n",
       "  0.11459009349346161,\n",
       "  0.7660967707633972,\n",
       "  -0.7983927726745605,\n",
       "  0.2929833233356476,\n",
       "  0.6888203620910645,\n",
       "  0.23778152465820312,\n",
       "  -0.6466079950332642,\n",
       "  0.1701143980026245,\n",
       "  0.395760715007782,\n",
       "  0.17905598878860474,\n",
       "  0.6261622309684753,\n",
       "  0.4910101890563965,\n",
       "  0.3575948178768158,\n",
       "  -1.0647172927856445,\n",
       "  0.2698685824871063,\n",
       "  -0.12187474966049194,\n",
       "  -1.1818437576293945,\n",
       "  -0.3489838242530823,\n",
       "  0.21053840219974518,\n",
       "  -0.23204903304576874,\n",
       "  0.3638300597667694,\n",
       "  -0.5968090891838074,\n",
       "  -0.057441595941782,\n",
       "  0.1112443059682846,\n",
       "  -0.291593074798584,\n",
       "  -0.2862301468849182,\n",
       "  -0.8793728351593018,\n",
       "  0.6892298460006714,\n",
       "  -0.7913636565208435,\n",
       "  0.5127228498458862,\n",
       "  -0.09938093274831772,\n",
       "  0.35494011640548706,\n",
       "  -0.6822109818458557,\n",
       "  0.17511703073978424,\n",
       "  -0.20586204528808594,\n",
       "  0.31522008776664734,\n",
       "  -0.5024538040161133,\n",
       "  0.4752412438392639,\n",
       "  0.47503960132598877,\n",
       "  0.8636654019355774,\n",
       "  0.4019636809825897,\n",
       "  -0.9821127653121948,\n",
       "  -0.5347175002098083,\n",
       "  0.07655124366283417,\n",
       "  -0.17329540848731995,\n",
       "  -0.33378615975379944,\n",
       "  -0.7636462450027466,\n",
       "  -0.18506228923797607,\n",
       "  -0.2044042944908142,\n",
       "  -0.5114434361457825,\n",
       "  -1.04177725315094,\n",
       "  0.2417168766260147,\n",
       "  1.1493103504180908,\n",
       "  0.3196822702884674,\n",
       "  -1.1377205848693848,\n",
       "  0.49643808603286743,\n",
       "  0.47277092933654785,\n",
       "  -0.7741323113441467,\n",
       "  -0.10644281655550003,\n",
       "  0.028499390929937363,\n",
       "  0.4810395836830139,\n",
       "  -0.025794070214033127,\n",
       "  -0.09291128069162369,\n",
       "  -0.24853894114494324,\n",
       "  -0.5108107328414917,\n",
       "  0.43960440158843994,\n",
       "  -0.21281054615974426,\n",
       "  0.6616883277893066,\n",
       "  0.012933901511132717,\n",
       "  -0.7300572395324707,\n",
       "  -0.8349596858024597,\n",
       "  0.5068705081939697,\n",
       "  -0.5877439975738525,\n",
       "  0.33972570300102234,\n",
       "  -1.0706830024719238,\n",
       "  0.05788709595799446,\n",
       "  -0.006646512541919947,\n",
       "  0.945362389087677,\n",
       "  0.4215848445892334,\n",
       "  0.3736646771430969,\n",
       "  -1.412192940711975,\n",
       "  -0.018384698778390884,\n",
       "  0.33942049741744995,\n",
       "  0.8478619456291199,\n",
       "  -0.6849127411842346,\n",
       "  -0.3425981402397156,\n",
       "  0.7211498022079468,\n",
       "  -0.11546867340803146,\n",
       "  -0.11989226192235947,\n",
       "  -0.14129747450351715,\n",
       "  0.26008141040802,\n",
       "  0.5184459686279297,\n",
       "  0.1494169682264328,\n",
       "  0.20914523303508759,\n",
       "  -0.293536514043808,\n",
       "  -0.9813276529312134,\n",
       "  -1.3408821821212769,\n",
       "  0.40968069434165955,\n",
       "  0.232853502035141,\n",
       "  -0.17887428402900696,\n",
       "  -0.8064798712730408,\n",
       "  -0.9370142817497253,\n",
       "  -0.8167057037353516,\n",
       "  -0.9076957106590271,\n",
       "  -0.17784062027931213,\n",
       "  -0.6639099717140198,\n",
       "  1.1732182502746582,\n",
       "  -0.5628486275672913,\n",
       "  -0.17304766178131104,\n",
       "  -0.037391968071460724,\n",
       "  -0.8997932076454163,\n",
       "  4.133209228515625,\n",
       "  1.0776386260986328,\n",
       "  0.13789090514183044,\n",
       "  0.09906791150569916,\n",
       "  0.22314929962158203,\n",
       "  1.1086506843566895,\n",
       "  0.019176756963133812,\n",
       "  -0.7972058653831482,\n",
       "  0.3454810082912445,\n",
       "  -0.4123588502407074,\n",
       "  0.6059165000915527,\n",
       "  0.2093440592288971,\n",
       "  -0.03261811286211014,\n",
       "  0.19128602743148804,\n",
       "  -0.2048986256122589,\n",
       "  1.3288987874984741,\n",
       "  -1.0208131074905396,\n",
       "  0.37542852759361267,\n",
       "  0.5316887497901917,\n",
       "  -1.2619023323059082,\n",
       "  -1.1552270650863647,\n",
       "  0.7879151105880737,\n",
       "  0.10308033227920532,\n",
       "  -0.03535159304738045,\n",
       "  -0.15949515998363495,\n",
       "  0.3826625347137451,\n",
       "  0.21668945252895355,\n",
       "  -0.8561273813247681,\n",
       "  -0.1706668734550476,\n",
       "  -0.25115272402763367,\n",
       "  -0.37538689374923706,\n",
       "  -1.0782787799835205,\n",
       "  0.3967038691043854,\n",
       "  0.7223605513572693,\n",
       "  -0.8813337683677673,\n",
       "  0.8240969777107239,\n",
       "  -0.29753756523132324,\n",
       "  -1.075225591659546,\n",
       "  0.4907808005809784,\n",
       "  -0.3166506886482239,\n",
       "  -0.29774975776672363,\n",
       "  0.7294722199440002,\n",
       "  0.2359035164117813,\n",
       "  -0.5984679460525513,\n",
       "  0.16493038833141327,\n",
       "  0.675334095954895,\n",
       "  -1.0304349660873413,\n",
       "  -0.23313993215560913,\n",
       "  -0.0821671262383461,\n",
       "  -0.3989616930484772,\n",
       "  1.010038137435913,\n",
       "  -0.5652366280555725,\n",
       "  0.3144356310367584,\n",
       "  -0.3301982283592224,\n",
       "  -0.9771621227264404,\n",
       "  -0.28078463673591614,\n",
       "  -0.10170087218284607,\n",
       "  -0.5724204778671265,\n",
       "  0.23880146443843842,\n",
       "  -0.4454110562801361,\n",
       "  0.738182783126831,\n",
       "  0.9331381320953369,\n",
       "  -0.5690971612930298,\n",
       "  0.4438735842704773,\n",
       "  -0.8533073663711548,\n",
       "  0.7612043023109436,\n",
       "  0.0969194620847702,\n",
       "  -0.19925881922245026,\n",
       "  -0.7010861039161682,\n",
       "  0.1847715675830841,\n",
       "  -0.5358695983886719,\n",
       "  ...]}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e37dd00e",
   "metadata": {},
   "source": [
    "### Scipy KDTree for Computing Distance & Fast Nearest Neighbor Lookup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ff164585",
   "metadata": {},
   "outputs": [],
   "source": [
    "vectors, texts = [], []\n",
    "\n",
    "for d in data:\n",
    "    vectors.append(d['v'])\n",
    "    texts.append(d['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b904d1b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.spatial import KDTree\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2c269e5d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(691012, 1024)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concat = np.array(vectors)\n",
    "concat.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8536adf2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8.35 s, sys: 80.3 ms, total: 8.43 s\n",
      "Wall time: 8.43 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "kd_tree = KDTree(concat, leafsize = 40)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "773da68c",
   "metadata": {},
   "source": [
    "### Generating Train Dataset\n",
    "\n",
    "we will generate three row for each index text:\n",
    "\n",
    "{'query': text for one key , 'neg': negative pair for each text index for key ['left','en','ms'], 'pos': positive pair text and query text for text in key ['left','en','ms'] except the key used in the query]}\n",
    "\n",
    "\n",
    "- Positive text pair threshold: text distance <= 10\n",
    "- Negative text pair threshold: text distance > 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "87851c1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys = list(texts[0].keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "bde36bf4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['left', 'en', 'ms'])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[0].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "67f93cea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# !wget https://gist.githubusercontent.com/huseinzol05/98974ae8c6c7a65d4bc0af9f5003786a/raw/5aa5257608b61e8fcc828e99fbd070d5ca7358e3/mp.py\n",
    "import mp\n",
    "\n",
    "def dedup(strings):\n",
    "    unique_neg = []\n",
    "    elements = set()\n",
    "\n",
    "    for n in strings:\n",
    "        x_lower = n.lower()\n",
    "        if x_lower not in elements:\n",
    "            elements.add(x_lower)\n",
    "            unique_neg.append(n)\n",
    "    return unique_neg\n",
    "\n",
    "dedup(['a', 'a'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f79f9f87",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "lower_bound = 8\n",
    "upper_bound = 15\n",
    "\n",
    "def loop(data):\n",
    "    data, index = data\n",
    "    with open(f'twitter-train-dataset-{index}.jsonl', 'w') as fopen:\n",
    "        for x in tqdm(data):\n",
    "            dist, ind = kd_tree.query(concat[x], k=len(concat), workers = 1)\n",
    "\n",
    "            for key in keys:\n",
    "\n",
    "                query = texts[x][key]\n",
    "\n",
    "                pos_indices = [k for k in ind[dist <= lower_bound]]\n",
    "                neg_indices = [k for k in ind[dist > upper_bound]]\n",
    "\n",
    "                if len(neg_indices) > 5:\n",
    "                    neg_indices = random.sample(neg_indices,5)\n",
    "                if len(pos_indices) > 5:\n",
    "                    rand_pos = random.sample(pos_indices,4)\n",
    "                    rand_pos.append(x)\n",
    "                    pos_indices = rand_pos\n",
    "\n",
    "\n",
    "                pos = [value for i in pos_indices for value in texts[i].values() if value != query ]\n",
    "\n",
    "                neg = [value for i in neg_indices for value in texts[i].values()]\n",
    "            \n",
    "                d = {'query':query,'pos':dedup(pos),'neg':dedup(neg)}\n",
    "                fopen.write(f'{json.dumps(d).decode()}\\n')\n",
    "                fopen.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6c18e51b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [00:09<00:00,  1.04it/s]\n"
     ]
    }
   ],
   "source": [
    "loop((range(10), 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "746022b1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▌         | 1235/23033 [1:04:06<14:40:37,  2.42s/it]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      " 20%|██        | 4648/23033 [4:13:59<15:07:06,  2.96s/it]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      " 22%|██▏       | 5062/23033 [4:31:04<17:00:06,  3.41s/it]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      " 23%|██▎       | 5363/23033 [4:46:54<11:49:29,  2.41s/it]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      " 41%|████      | 9370/23033 [8:29:49<8:16:28,  2.18s/it]]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      " 43%|████▎     | 9959/23033 [8:57:39<8:34:32,  2.36s/it]]]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      " 45%|████▍     | 10333/23033 [9:20:16<11:32:04,  3.27s/it]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      " 45%|████▌     | 10472/23033 [9:25:32<8:43:38,  2.50s/it]]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      " 48%|████▊     | 11001/23033 [9:46:41<7:35:11,  2.27s/it]]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      " 48%|████▊     | 11018/23033 [9:55:52<8:47:36,  2.63s/it]]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      " 49%|████▉     | 11320/23033 [10:22:28<8:34:38,  2.64s/it]]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      " 56%|█████▌    | 12857/23033 [11:23:01<8:05:00,  2.86s/it]]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      " 56%|█████▌    | 12891/23033 [11:47:03<14:24:53,  5.12s/it]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      " 56%|█████▋    | 12993/23033 [11:50:50<10:07:07,  3.63s/it]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      " 58%|█████▊    | 13396/23033 [12:16:08<9:19:52,  3.49s/it]]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      " 61%|██████▏   | 14156/23033 [12:43:47<5:54:26,  2.40s/it] IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      " 91%|█████████▏| 21036/23033 [19:13:40<1:56:09,  3.49s/it]"
     ]
    }
   ],
   "source": [
    "import mp\n",
    "\n",
    "mp.multiprocessing(range(len(data)), loop, cores = 30, returned = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa5d45dd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}