4_Crawled articles.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from article_lib import *\n",
    "from mitie_lib import *\n",
    "from processing_lib import *\n",
    "import pandas as pd\n",
    "\n",
    "article_folders = ['crawled', 'fakenewschallenge', 'KaggleData']\n",
    "working_folder = \"..\\\\Articles\\\\\"\n",
    "fakes_dataset = pd.read_csv(\"..\\\\Data\\\\fakes.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Started processing folder ..\\Articles\\fakenewschallenge\n",
      "Found articles: 49972.\n",
      "Processing batch #1...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #1 finished. Time spent: 52.8900001049. To finish: 26339.2200522\n",
      "Processing batch #2...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #2 finished. Time spent: 53.6630001068. To finish: 26670.5110531\n",
      "Processing batch #3...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #3 finished. Time spent: 47.3190000057. To finish: 23470.2240028\n",
      "Processing batch #4...\n",
      "Found 1 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #4 finished. Time spent: 56.5099999905. To finish: 27972.4499953\n",
      "Processing batch #5...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #5 finished. Time spent: 53.7320001125. To finish: 26543.6080556\n",
      "Processing batch #6...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #6 finished. Time spent: 81.3350000381. To finish: 40098.1550188\n",
      "Processing batch #7...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #7 finished. Time spent: 77.0620000362. To finish: 37914.5040178\n",
      "Processing batch #8...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #8 finished. Time spent: 119.028000116. To finish: 58442.7480571\n",
      "Processing batch #9...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #9 finished. Time spent: 65.0220000744. To finish: 31860.7800364\n",
      "Processing batch #10...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #10 finished. Time spent: 36.9900000095. To finish: 18088.1100047\n",
      "Processing batch #11...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #11 finished. Time spent: 45.7319998741. To finish: 22317.2159386\n",
      "Processing batch #12...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #12 finished. Time spent: 61.9750001431. To finish: 30181.8250697\n",
      "Processing batch #13...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #13 finished. Time spent: 43.6050000191. To finish: 21192.0300093\n",
      "Processing batch #14...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #14 finished. Time spent: 68.0729999542. To finish: 33015.4049778\n",
      "Processing batch #15...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #15 finished. Time spent: 106.376000166. To finish: 51485.9840803\n",
      "Processing batch #16...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #16 finished. Time spent: 58.3710000515. To finish: 28193.1930249\n",
      "Processing batch #17...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #17 finished. Time spent: 34.3499999046. To finish: 16556.699954\n",
      "Processing batch #18...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #18 finished. Time spent: 56.8289999962. To finish: 27334.7489982\n",
      "Processing batch #19...\n",
      "Found 1 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #19 finished. Time spent: 55.2849998474. To finish: 26536.7999268\n",
      "Processing batch #20...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #20 finished. Time spent: 96.9519999027. To finish: 46440.0079534\n",
      "Processing batch #21...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #21 finished. Time spent: 52.361000061. To finish: 25028.5580292\n",
      "Processing batch #22...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #22 finished. Time spent: 77.6500000954. To finish: 37039.0500455\n",
      "Processing batch #23...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #23 finished. Time spent: 72.5559999943. To finish: 34536.6559973\n",
      "Processing batch #24...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #24 finished. Time spent: 48.4029998779. To finish: 22991.424942\n",
      "Processing batch #25...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #25 finished. Time spent: 51.7650001049. To finish: 24536.6100497\n",
      "Processing batch #26...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #26 finished. Time spent: 40.5469999313. To finish: 19178.7309675\n",
      "Processing batch #27...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #27 finished. Time spent: 49.2209999561. To finish: 23232.3119793\n",
      "Processing batch #28...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #28 finished. Time spent: 42.6880002022. To finish: 20106.0480952\n",
      "Processing batch #29...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #29 finished. Time spent: 95.1080000401. To finish: 44700.7600188\n",
      "Processing batch #30...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #30 finished. Time spent: 110.049000025. To finish: 51612.9810116\n",
      "Processing batch #31...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #31 finished. Time spent: 73.4690001011. To finish: 34383.4920473\n",
      "Processing batch #32...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #32 finished. Time spent: 52.6419999599. To finish: 24583.8139813\n",
      "Processing batch #33...\n",
      "Found 1 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #33 finished. Time spent: 65.4559998512. To finish: 30502.4959307\n",
      "Processing batch #34...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #34 finished. Time spent: 72.5. To finish: 33712.5\n",
      "Processing batch #35...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #35 finished. Time spent: 132.719000101. To finish: 61581.6160469\n",
      "Processing batch #36...\n",
      "Found 1 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #36 finished. Time spent: 122.394999981. To finish: 56668.8849912\n",
      "Processing batch #37...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #37 finished. Time spent: 58.6979999542. To finish: 27118.4759789\n",
      "Processing batch #38...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #38 finished. Time spent: 62.5739998817. To finish: 28846.6139455\n",
      "Processing batch #39...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #39 finished. Time spent: 60.9760000706. To finish: 28048.9600325\n",
      "Processing batch #40...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #40 finished. Time spent: 133.352999926. To finish: 61209.0269659\n",
      "Processing batch #41...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #41 finished. Time spent: 92.4700000286. To finish: 42351.2600131\n",
      "Processing batch #42...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #42 finished. Time spent: 111.733999968. To finish: 51062.4379852\n",
      "Processing batch #43...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #43 finished. Time spent: 80.2920000553. To finish: 36613.1520252\n",
      "Processing batch #44...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #44 finished. Time spent: 52.6630001068. To finish: 23961.6650486\n",
      "Processing batch #45...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #45 finished. Time spent: 134.990999937. To finish: 61285.9139714\n",
      "Processing batch #46...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #46 finished. Time spent: 72.978000164. To finish: 33059.0340743\n",
      "Processing batch #47...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #47 finished. Time spent: 68.2999999523. To finish: 30871.5999784\n",
      "Processing batch #48...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #48 finished. Time spent: 101.596999884. To finish: 45820.2469475\n",
      "Processing batch #49...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #49 finished. Time spent: 105.239000082. To finish: 47357.5500369\n",
      "Processing batch #50...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #50 finished. Time spent: 48.5939998627. To finish: 21818.7059383\n",
      "Processing batch #51...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #51 finished. Time spent: 86.4930000305. To finish: 38748.8640137\n",
      "Processing batch #52...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #52 finished. Time spent: 163.217000008. To finish: 72957.9990034\n",
      "Processing batch #53...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #53 finished. Time spent: 120.414999962. To finish: 53705.089983\n",
      "Processing batch #54...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #54 finished. Time spent: 79.8059999943. To finish: 35513.6699975\n",
      "Processing batch #55...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #55 finished. Time spent: 50.7950000763. To finish: 22552.9800339\n",
      "Processing batch #56...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #56 finished. Time spent: 88.8009998798. To finish: 39338.8429468\n",
      "Processing batch #57...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #57 finished. Time spent: 72.6509997845. To finish: 32111.7419047\n",
      "Processing batch #58...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #58 finished. Time spent: 62.5230000019. To finish: 27572.6430008\n",
      "Processing batch #59...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #59 finished. Time spent: 40.253000021. To finish: 17711.3200092\n",
      "Processing batch #60...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #60 finished. Time spent: 52.9779999256. To finish: 23257.3419673\n",
      "Processing batch #61...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #61 finished. Time spent: 79.2699999809. To finish: 34720.2599916\n",
      "Processing batch #62...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #62 finished. Time spent: 58.8310000896. To finish: 25709.1470392\n",
      "Processing batch #63...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #63 finished. Time spent: 48.8619999886. To finish: 21303.831995\n",
      "Processing batch #64...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #64 finished. Time spent: 95.3289999962. To finish: 41468.1149983\n",
      "Processing batch #65...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #65 finished. Time spent: 80.8460001945. To finish: 35087.1640844\n",
      "Processing batch #66...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #66 finished. Time spent: 52.2079999447. To finish: 22606.063976\n",
      "Processing batch #67...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #67 finished. Time spent: 66.4440000057. To finish: 28703.8080025\n",
      "Processing batch #68...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #68 finished. Time spent: 87.2880001068. To finish: 37621.128046\n",
      "Processing batch #69...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #69 finished. Time spent: 146.219000101. To finish: 62874.1700435\n",
      "Processing batch #70...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #70 finished. Time spent: 64.2939999104. To finish: 27582.1259615\n",
      "Processing batch #71...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #71 finished. Time spent: 81.881000042. To finish: 35045.068018\n",
      "Processing batch #72...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #72 finished. Time spent: 77.9340000153. To finish: 33277.8180065\n",
      "Processing batch #73...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #73 finished. Time spent: 23.4679999352. To finish: 9997.36797237\n",
      "Processing batch #74...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #74 finished. Time spent: 67.0829999447. To finish: 28510.2749765\n",
      "Processing batch #75...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #75 finished. Time spent: 46.753000021. To finish: 19823.2720089\n",
      "Processing batch #76...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #76 finished. Time spent: 48.2409999371. To finish: 20405.9429734\n",
      "Processing batch #77...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #77 finished. Time spent: 121.692000151. To finish: 51354.0240636\n",
      "Processing batch #78...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #78 finished. Time spent: 66.3789999485. To finish: 27945.5589783\n",
      "Processing batch #79...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #79 finished. Time spent: 72.3040001392. To finish: 30367.6800585\n",
      "Processing batch #80...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #80 finished. Time spent: 37.6410000324. To finish: 15771.5790136\n",
      "Processing batch #81...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #81 finished. Time spent: 30.4040000439. To finish: 12708.8720183\n",
      "Processing batch #82...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #82 finished. Time spent: 54.2919998169. To finish: 22639.7639236\n",
      "Processing batch #83...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #83 finished. Time spent: 82.6649999619. To finish: 34388.6399841\n",
      "Processing batch #84...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #84 finished. Time spent: 51.2620000839. To finish: 21273.7300348\n",
      "Processing batch #85...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #85 finished. Time spent: 76.0650000572. To finish: 31490.9100237\n",
      "Processing batch #86...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #86 finished. Time spent: 34.7509999275. To finish: 14352.1629701\n",
      "Processing batch #87...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #87 finished. Time spent: 54.9749999046. To finish: 22649.6999607\n",
      "Processing batch #88...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #88 finished. Time spent: 27.8350000381. To finish: 11440.1850157\n",
      "Processing batch #89...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #89 finished. Time spent: 57.6869997978. To finish: 23651.6699171\n",
      "Processing batch #90...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #90 finished. Time spent: 63.6979999542. To finish: 26052.4819813\n",
      "Processing batch #91...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #91 finished. Time spent: 56.746999979. To finish: 23152.7759914\n",
      "Processing batch #92...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #92 finished. Time spent: 61.6430001259. To finish: 25088.7010512\n",
      "Processing batch #93...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #93 finished. Time spent: 67.2760000229. To finish: 27314.0560093\n",
      "Processing batch #94...\n",
      "Found 3 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #94 finished. Time spent: 44.7699999809. To finish: 18131.8499923\n",
      "Processing batch #95...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #95 finished. Time spent: 41.1490001678. To finish: 16624.1960678\n",
      "Processing batch #96...\n",
      "Found 0 fakes. Results saved in file: ..\\Articles\\fakenewschallenge.csv. \n",
      "Batch #96 finished. Time spent: 102.805999994. To finish: 41430.8179977\n",
      "Processing batch #97...\n"
     ]
    }
   ],
   "source": [
    "#for folder in article_folders:\n",
    "folder = article_folders[1]\n",
    "process_folder_of_articles(working_folder + folder, working_folder + folder + \".csv\", fakes_dataset, batch_size=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}