From 063d9225c9f5c8c6cb7d72e4481ce0d81a895b79 Mon Sep 17 00:00:00 2001 From: rishisankar Date: Mon, 27 Jan 2020 21:19:31 -0800 Subject: [PATCH 1/2] Added wrapper functions for scraper to pipe errors to email. --- menu_backend/menu_backend/settings.py | 19 ++++++++++----- menu_backend/scraper/db_insertion.py | 35 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/menu_backend/menu_backend/settings.py b/menu_backend/menu_backend/settings.py index 8acd7b7..ceba2bb 100644 --- a/menu_backend/menu_backend/settings.py +++ b/menu_backend/menu_backend/settings.py @@ -45,13 +45,13 @@ ] CRONJOBS = [ - ('*/5 * * * *', 'scraper.db_insertion.insert_activity_level'), + ('*/5 * * * *', 'scraper.db_insertion.insert_activity_level_wrapper'), # everything else runs three times a day: one at 0:01, one at 10:00, one at 15:30 - ('1 0 * * *', 'scraper.db_insertion.insert_hours'), - ('0 10 * * *', 'scraper.db_insertion.insert_hours'), - ('30 15 * * *', 'scraper.db_insertion.insert_hours'), - ('0 3 * * *', 'scraper.db_insertion.insert_slow_scrape'), - ('1 * * * *', 'scraper.db_insertion.insert_hourly_scrape') + ('1 0 * * *', 'scraper.db_insertion.insert_hours_wrapper'), + ('0 10 * * *', 'scraper.db_insertion.insert_hours_wrapper'), + ('30 15 * * *', 'scraper.db_insertion.insert_hours_wrapper'), + ('0 3 * * *', 'scraper.db_insertion.insert_slow_scrape_wrapper'), + ('1 * * * *', 'scraper.db_insertion.insert_hourly_scrape_wrapper') ] MIDDLEWARE = [ @@ -146,3 +146,10 @@ 'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder', ] + + +EMAIL_HOST="smtp.gmail.com" +EMAIL_PORT=587 +EMAIL_HOST_USER="errors.bruinbite@gmail.com" +EMAIL_HOST_PASSWORD=os.getenv('ERRORS_EMAIL_PASSWORD') +EMAIL_USE_TLS=True \ No newline at end of file diff --git a/menu_backend/scraper/db_insertion.py b/menu_backend/scraper/db_insertion.py index 49bf63a..e7c847d 100644 --- a/menu_backend/scraper/db_insertion.py +++ b/menu_backend/scraper/db_insertion.py @@ -2,9 +2,44 @@ from .models import * from datetime import date from .scraper_thread import Scraper_thread +from django.core.mail import send_mail days_to_scrape = 7 +def sendLogEmail(err, funcName): + print(funcName + "() threw an error: \n" + str(err)) + send_mail( + 'Error in menu scraper', + 'Error in function ' + funcName + "() with the following message: \n" + str(err), + 'errors.bruinbite@gmail.com', + ['errors.bruinbite@gmail.com'], + fail_silently=False) + +def insert_activity_level_wrapper(): + try: + insert_activity_level() + except Exception as err: + sendLogEmail(err, "insert_activity_level") + +def insert_hours_wrapper(): + try: + insert_hours() + except Exception as err: + sendLogEmail(err, "insert_hours") + +def insert_slow_scrape_wrapper(): + try: + insert_slow_scrape() + except Exception as err: + sendLogEmail(err, "insert_slow_scrape") + +def insert_hourly_scrape_wrapper(): + try: + insert_hourly_scrape() + except Exception as err: + sendLogEmail(err, "insert_hourly_scrape") + + def insert_activity_level(): activity_level = scraper_for_activity_level() ActivityLevel.objects.create(level=activity_level) From 4910e2689c6fcc2aa41913126d71e0f5ccc3b84e Mon Sep 17 00:00:00 2001 From: rishisankar Date: Mon, 27 Jan 2020 22:27:36 -0800 Subject: [PATCH 2/2] Updated menu dev env to hold error email password --- BruinBiteSecrets.tar.gz | Bin 5372 -> 5428 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/BruinBiteSecrets.tar.gz b/BruinBiteSecrets.tar.gz index d2f172bb1678adeabb7afc303044b20c708ded26..e1e72682e123671bf114ea9c80c73aab7f0b2cac 100644 GIT binary patch literal 5428 zcmV-470c=XM@dveQdv+`0M6-GeCd@kkYaOrY9DfGkIdO)Zo4bEVBjd>ze(|y%|v)* zwoFOdyU`DfPTU=Bzt1lOI`9a)O^G8AQZ=bm2GF8HzGQy4c^HKc-t`8XmGnTWO7zG6 zZ>uAXV%#~@_)57H{gi|bkMiL6X3UX58F}^}RN*fz*L>@-wo2cgH!ZA>iLaWYk(`7{ zx$hQqv9ka@ImAXI2_}5Sl6r?UyhvjD5=D+*4>54pBNj1Grd8uS+tIQpVOC-%IWD)- zIV3@W`L^rqV*sxXCC3sincBHJtneg<nIo<8Il z8;Kbfl%;@EKz`jNP%^2eXKU|G^pB>n;Nb`Fc*H#)?Tzm3*j!dR*`b*~$?bzl=LF3Q z+?j+vAC?>v;%%Bal^z5%)=X6tk&*Ufi%3pg=qsS|lSJ7f>&Bi0vw}mn1NRE|UWG7X z34vuA!T~x6H2huHPA!G&rhZ{W_x`+(mm&90iNTNN;{NZK~YY79+4n z*3Ti|zl-k;2}Kwz`5lOr5<-eGy1i}Z_sN)Ffxf0AM*t%D^UbaeAtP7MuNXTAZ_SLh zvXd!`VFK~cN_UspNnk4H=f5&^msplQD36~jAsPk-;~^`U5xK2Pf?5474)%^>d>f_k zi$7b=2HBNv@7rl!%@q!nQ#Xbn#c(+}D0c0*Z(4DWGTdsI^bC45a)3N$<3Xsm*%Nh9 zDX#u342GwXEakBPO6(CQciE+YF2gDma$7sI*fZsb@tY9cy(q>LP$%7@kn^|}*)9!w z6u#D8X9uI3dTllBCESPtsP43+RkA5p|Qag7-T7?=pQ53&wIDt0h zMOHUn^Ylxu#*lpVj+7iq!Y!C6LU5H~^hOv`r^?L4B&PQAxdT7$G<+Hd>L$S8NYuPi z>eJn)NfBQn7j7v=Lv0l08M7AJGrDNXtu$&1^-Eq792a1C9B*NcB=jEN1A;DVYa{esppKD&qWcWP z_};C#&b6oOTyz(TR0SPgQ8h+J#?Z{gP6@k{9>BwRWS)*xTOrKGS;-_8{&^+DcEame z7#@G)`RF4WL7e8t?GnqbLbcaCY$%>LWvb4JT2~D2GWgUEJT+I)=>#4crr_4ZG;r%t zB<46=)5WEeJ(onl7)Ey`gjOT-Ruj1qN@1`9Tvut+t#5jP7qE%35axK*`eo3bFvCrj zXPWW}vhUvL4!6L5w;)v z0V+h&CPJW_L)Ak?6e*i0`8G8hXkY?vx*h28^#Mu5z$6ez88HOtTZh~C+w*x9r*$hY zot<+G%;(WjCs}0B4+gS5N-KUpRPy2e`d-aPsH5b-jmly^l`hD=h{q_bl!{>5s#&Rc zZ5ICm8H^ka839-?3a<=4k$6QOMH#(`HOxi)v)iL``qcb7l#DI8EiB{VnWwR>`howY zH3!S~*33F1u#yiQU4YBX`hJ-vz}Cm!gFAoS>V?@>3M;7w`Lz&)YIR4`$Zc^L6zyPbH-NV&Lmr!3U>qi3nz+4oTiK&kx7 zw9exq#?QvRxK_d#Mhl2G%>@S$jK%H_4J}XK<__hnsGYnSy`;D_3WL8nisUsd3j1+( zBU5}vT*ss2>*yZLE~MJE*^x|I!0o&D5_Aj<}8Qhg^r5sw8&?jaF(j|K9Pi=&ZG4nnX67+uCeAW$ zYV&i}Xe3^m4H9$zkB3TypzAY{i#L(IeeQe?hn)zO0>h=nR>Dz9m0<(&CH5p^5wtkv3AyX03o-5>dKgD>f~q9`;dYXy|W$6E{T!J8#1!hD}5H1Uc0 z9TS=JF?TSj&GWh~$n4GpCuPRB^6f)+R-q;Puh6$ahr(jjhv&|OopS_oPUWXaH`JVJWft?kX$pG_Pw8>v z?wCDviH?X{ry(+j!gpR6UhSL*Ys7U}eU|B--T#WHfSrnsP@eKR23fH3|9)*@1^=(PTK-Qx{XqLH${m& zh+?w-4k`@_-{`n>#bI*DOL{Jkx$IUIR`-w$fqx=3j-+Ke)QLJU4 zzdJjg=e`-C!ymUnC;?HOiJQXD)mqxP@9G0-k^2Et{C6wJC0QUU&xmZrjmYO@)Ut(e zSpdibM?Vw=iliF`Sv(rK^%_GB4`!QdUFB`ypVdfFgrG+(+Sy4~Y}s8bvH=}da;u>M z!&2r93qkji@OW)I{=gwYKWB43M~~nNO$sWqBN6gnf7%8Fl{dRP;xU=i^g6PyWBI-r z4UWm`2>z4a%8|xV(P_xIwT<3$A)A1htn+gf=k!?MP%)%kZB-a~$+m@t+G(C#i}h&C z=yhOfM5RYBh+npyAaD!6%^l$?g~J<0dY=owxd_5|8zV>nty{S`Wy%;Z=Zk0AFViJF z^u(PuyO8!wRv9OTv~qMXDH&x9WRc0>{#qHKUrb_Dz;bb0X0nQg=sqkf&0|h|aRAmR z{Jr3K)|@B~<)WhEI7`)G{jFbofF)Fw`1V|y^6XH!=O)oJqDLbQ0+J(OFjw6@c1g(GAdEP1Z_Oxoz6rD z(N@=#G3gC2Y65K~nfEIga~eISV0Gn5Ulvq-esfK1s3&L48v${~0|#>O%8HS&_!Ga4 zug9p1uyah>coB-0%$W0DanA{*(Cp*@PTs(wtC~uP>WOy?Ki;Q;Q;`(`PPn}Ff$%l< zN{|&c28Q1Oim1BY2Sem(tMKfwQtwdTl1t7)>nDNER?%HQXm0I&Ab$rRT~eZ<)}m8A z0~LX%m!ky$aC-VUpmC7L7e7s8u@=LT)OC`&3^Do%4QnJVWid%D`R*lr^2}@GFpp)E z_5_X^_;XI?FLpjpgfU;1s$_l~rJ`dP1OrGHI{Onz!7&XbCL6`0-H z*GCI}a9+kROyKLq@`?5t=|chSCBCEPHM!`lxkSf?sK!!oz=%jnSWl?`pPPCU1Zgi> zpEyamb!Y__F%<`9$0dSetj;oQRT_*886Tv zLkvSn-(0&fF5}EC752)vK z>--Ij$fGh_gNf*Z>^my&C|8r7;(Ei{=Z2?I?BK;vZ3}(|`!qkvIHVQ%N>YFq*t5=R zo3B{DZUoCcVKp>j!m)@`hji3wU7VpH(s6R2hyBb3vQEH0^5rfxhc<@(g+V&`SGc zgo7jsGsjAeP0UheGP-06)p5?6I@t(`D>8=<>>d1AG0&tStkRC3_SZCJ7b(6wnkh`d zJ9>zB2!*XekedR6W3)tJ&7q%YO7IK&tCviFfLz_#9UfK>%38(!ffgv?wE9)eeG!8E z7w;K9h91%xO2(>h(z^}jlfY3(QJ?xj!wm-0M}N z`LUdzQ%2q0_tXp~Ob(7ULY6(47pRuyUG_;pw0|LPVrHvz{igY$gZCdtpYQP4g8e9q zM0F`zBU*%r5w-MBd=_rhw9QT5)^C#Y1AGC`a?979wjS9hNv_q78Ee!%U+-Xl2CyUH?j$0*cFWuPEZ^N?t|Z5J^K*n5Gl{*D9Yn%VD&% zy5&@aCn6}M$w(%dfjFxj9+~UfD9r`ol=7f;lLF z$0L< zd`s>3t3X5C)o9eugpK+)^`7+}n4D%0`rqNvJAe^*?KoM`zr}rcrD->QH8zgQkFFxH zvisZx<{kg+MoHX_=X!gCn%R!BDG`{%=Dn?31qQ~dMX$y1XbO$-WF`S?hw{ObqnVuG zRYwVHT#n4?;{rS*UI|eH`lUpyL-)w2 z3YW{IpIIb3(HO~+=dI}l=LjHb=K%8jII#hqiLA^R%88TF1{gbGboya8@9+>e=EBcB zL8?^Vjctg`L!BfKgrNrswqMU4KMI5@Q#I~b@%2n3tGgO*E}f45caUY`Glz?6a(SO8 zPd*LY$pVrJ4a8ZD^lA>sx@}gNjC-p)yQfY&A{zsQXA2m^PCHWjaEObbs&rqy+FWb^ zo&iR^b1F$SbrOGK6%!EH{glrt1VCg{56wP})%q28*wnTalZ!?OC`jDFmmt>LqGaZO zU<(D(9R$ZyTRQ0Y7_){^i;P6Ars91XFG&ckFj|sN(=oo!AUV3TBdSPLJeqP2*4J4H zD$7*Azjb+@TRBKet~lCa+Hn?9i)V)DJh$Y_EexJ1RjtAMdL~uE(piXKnZW((GjG^1 zQ;-c;y-`-8_kCWIw}3OHNqWX{CI@)j_%qz?CVaZv;938*VF(1D+;WGkN!gK;Yo5yV z3y^Rn?Xhf2ag2&3F5O0N2X#5Lf&2y&XRl4DfZY@jGcMep{d5<>;C z-_f>mD#+R?@|wH_vG^pPmw}Ubr)b9qpIk7=?_>FZsZ6x4#0EZI3eULo(nKhA!B0hR z#(fXhDg5ddb;iolr(~<-=gkMG_B|JInyIZi)#;dM?uzNi5Bm`eZqArRUa?MMQZF~0 zfp_XS(?JoalPKla(sfNRAUeNL6gMQDVMp4vTzc{zDG&Bz*G7I48)LWi1u2wRi^$sp ey?JG-!W&LSvfapn%%0gFf0-jjKQyi2wN9=hB=rzd>9onvXR0MSP`)*d zhW}A>A!YaOBR=VUI5}AM26F`Y;~{hf{x@aTV$+!KIWAFtcM*CS=6weQmUE^A^ZkoP z_)pNUyG!f-zF!>4@nY}$k)>7CzmOzV%UXPFeb*Onm#*?~@(hm!e6@nZswSFx*~6VO zMK<-lh>7WC1Zq0!xe1 zXLKILP}uGVOrW6m-b;EH5@?R$r`rXR52YS*qh+x*W!}tGB3=3=+tJ8hEN&-s5ga;(~flE>f@Fp`mr zT!GdF-kRX80E_AQD21wtDNo6^lvHhRTp41Ka*Ftp+O0vt&eNj(bnWcR`1~xfs15h- zhRls`o*F8mUP$0;_{r*KV99KSaT^f%r7eKSA{q~*ty5QtRbbWrv<{tM2nYx3o2rw7 zUBUh!3nezU?#I3HkuD#g@0v4UwLzTC&x!UVOxN)bGtk2bUT^Nf=&5xIEnUKJ)K%Q8 z!V=2L4xDiFu{g8CDI|}(>N$m=P~<9{i##YHsM7@32c*Ol=&lK`Nm5-?QJiWwdy<|% z+NX;&x}aU<%!YaW2y(mj~%wA270BT{5%vqC#?cfe+ zF-!yJ4wd_uI0%6zFlQ#tNVp`0itN~9pdo#ejf@%^1e-VFn__lk=;SEwd0Vt#~M>B=nY%U*AZa%#7TXD6gOW?h21veae z&T~ilHD8>;`1bHX=YdwT#{`zVT=^8`+ptCvQvRFf3QBb}QNY}acFAqaU!mxP$OaS! z8}=@JUhVQ#Can4f{Rykv~?q0Du?Ba^P;G5mIY!lS&t4bs9iT(+^-e$s10Xb$c4R01)&NBz6! zpn#&#TZOILL3*xB;`28S`dm@0DAO@19v`2^@KgLGtHb6UIG`PG=B`XSG(KOQG6@?$ zy?j}j;fk*%(VD~r=q=0~{dq<9gEv>B3;;0WJ4vx<5D+m56!p`aALxz6Fs!tzDaO8oKY z1$k_R3%2-=HS70i;B`+;x-<`^cgvjpA{FqD4cZY{<~38n<$2L}Wr^5Td6iL)7s3NL zF@F*vr>r=NojM~krt3y$EPC~Z$o_Lu1Qnmcf&$sUam$_55pDPGQ@_d>{Ih2F1`^P} zIqvK!;EQ-bc+C@@V6zNHd z@-n?*EcUZT++#uDfkdFv!N9j+&a~SUr6$O81Ga z-CsH^%kaw14|lQB7CCgKe1VPIgF^ANjzzLo-F*-SO!~)q^ttWnV^>E+8~4)=Ae?G+RBzR8VC7`Yd4 zXn_ZJ^He!?bHPQx8T5skSF0)$W?qB#iR+fbHU`svGRTEa+p&@6J32kmNh` z!XO$o56waUBLbXjyz>tl4yZ6xN%#LRxrx#jgCEY7_ejpj`sJAMq+}Neq9s6+KQm7P zkp+e9%5*h8X*6QS z@V>jf_i-6OLd8Ra`@x3xTl+Tdr<2an9EwA72lK_|x>u2~u8M+JRd}GV+ajW%nlqh8 zbbX$Q9|GM2mYn3Ji2>B9o$bw32WY92y<<0l#%==hlmq)Jvpu=Iilp>(*x9`N`r0 z8pRCOk4N|7Vvya!Q}7YMII>^*8jbhgG(Ts&i8cz!ff2Jw^rJo)iejPu_K8K2rkE}Y z!)a27hwKc3eYdq$R)DaKEmDFwRrdK2qTtpSU1(Gl*nd+1X4@uS4#FT@&FG!0$fS1_ zf00MY*v8Gle@!+;$fk``c9o9iPQs7EuXGN4K_P?kIl!bZ74~_cDv@@6%g#*&?tWQI zO!x)N6))?auY!o6S{dLo5Um}HT1FEwvhj}(JGl%mSmbtg9#Lp&uPK=Iq zQz+y6UtYp^g2U2|pkiH|h~|+G*xfvk4i0_SQY_sP;LplF3@DtQ;mhEUEX*;GiKAlB zT<33;HhIfP&3Fq-W-i2k0B?2?$s&^;S?qgE)gH{B6O`kIPuWp+5RrInxSUHjy^ODf z)vSW|enI5`-I_Kb_|0{2qFuz9e%d@|IZPtP6L+`m!ueva$sg?}NeWoEK=E-YrZ_-| zQ#R#vuB%Uw`U|ifQI0=&#`&l{?2GuAj8Ab!Z~dJ0XqIfzu}MB6PQ0KD_XCO1f5R;d zc{G)dmClaB3QSa)(ye?SOc}t}R3#Dzuu>F$kI{IRXlEd&%Ou&A~1efZA znIuv1B9v!XIMkQuwUBpf>WCd~OB=o?)!;pKQ6lwI32b8v%R)38R{2A#qP|-V2e2ig zn&>fb*s0?$Y>+fdGG-H)&DrEVu)6&hk0=9^$I+X$jD@zDX4=Q)26y9oY|w}_P9Au{ zUrVTfAJpH~a>64Wd(P!LH}K#8e5V=}2WQ4=W7~4M zFP^e;vjL28X(rd-8L2I76aQAYOv}v`%!?&P)V|**A`AW-^r^GMV zNe~Mz`>F{APPBP%Hy+2K5L@laKY<)`FAEV;6Ip9nCc1fF1`zOGFJ>k3xqI@t)bAIJ zyg9v=0h=D|Xik*-LW-Ak0akB`mgQt@E2E#J7v||6b?19$civmU)&&@K(ssZb8tdQX znfiqWA<1qsgz>xFU>r^L52F-5eC*F^f_qZil**zK1oZXbUg=LYUzRO5opT7_Ft}o* zQqe7l(qjj$x}HsOR4q+uta)f6_IV>uc;qfQXM!(LwsNn~ZA0a!0G?E|g6g|R*Dv+< z)%LPvA1W>C&?V)nSm7TUO$eok@9?WJAaeFt88AhSEF>7m@ldSNK;C~;Ns%5t;cKT; z3NcSBm?yPukg{v^pHK)?M|ev0lnQllGRz&837@XsSMHaO&UuqE`7bFZNBjv-^-txG zPBc1^Li6K=IO;XUuov=~_DoM`Nsm~3F2P|;6N*D1X!T;eVGE8R_FW4IP!mY=*hX!t zcIP~*RN8q!ejcifYODFNjbi4CpphtLJy!MMH4p(@SuiA2WvJ~CS+^KQ@uO3>rEpYu z5!Ylzs%ncWK6OQ>arbeMgK*o-Lxfd|6(@W!Nn?V0^8xL&Q*{if+H7s)^fQZydg!?0 zGxzZ}hV*wL`^Ap4aP}i^as4_!+9BvYS8KLc)3OjS&g|Flp_W7}Vp3-V_&FW@UX~nmust}4!*n3@lCxQHM-9p1KpoXfo@)(aA;)LU+8v`XbG*y- zDMlWtyV_hY2&OCHcIx<$VThOcLoyMgWr~$u|HqL0wvzIfv>QZIsGf)O_qDMZ0C`}4 zfciiqj8g|8E&{+4wYLWQT=K8ZQ_Gjl^SmMNEt$(PT=@-a>qdvnbY?~w+26m$>l36H zjt@9cRpqQ-vlH2Gq<}r#DHvOAHvH_me_0gF_EACFR#lM$APgm;8(&HP`z8LjrA8qd zSqM$ejpE%TbUDFE-P!an0|npVS6}c}@R%AraH45$@5vKI750=HfFW=Y9#r&IXFH!& z22JgnwfOSlgKJ08f|bUG83+UCdFY2TzS>>7;`H_JUS91y@EST3REoK9k+LJR%y^ksj+kH&A_DftN+#dM(u z*A0|~@*&sv57lN=>c*EU!ZW*A;YY(u`KpY-bnzFCD`*iwm4FaKcJoFXRAA4W2t2z) z#4~ot82E-+zq(k-(i*Q0#4MRgn0s0eVgJM75Ci%axf37-yr$)r~ zVfUejz^{k5{NS>N3tA5ivITqKbE=O&DK|5%D6HQXRaN|7Q@5c(Evy%VF-7}i`t8Mj zu=bZ;0rTrgwNJzZ2WI+4q>15Pw6c>bCZ&8d(bK1sF29TuYxLEaE*Wx^FWA@LVR~`h0;W zOjbs~o7#ft!@_36k8q~jSS0iayq06S)3kb?X({%CxJc*NzK>}_`*`29e;n({bNRCw z4tpILq4#^D!3qeVnMoMWw$B(UPM^n{(?XSm(R}UjuUo)$j>n(k!Nd*7ddiwDQaYij a4Imz8qK(N+V9Gcy(408pnjk!Q#rMP%hjB>&